SHARE
TWEET

Untitled

a guest Sep 20th, 2019 195 Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. 8893 org.apache.hadoop.hbase.regionserver.HRegionServer
  2. 2019-09-16 15:37:03
  3. Full thread dump OpenJDK 64-Bit Server VM (25.222-b10 mixed mode):
  4.  
  5. "IPC Client (1552133633) connection to ip-172-20-113-124.us-west-2.compute.internal/172.20.113.124:8020 from hbase" #14627 daemon prio=5 os_prio=0 tid=0x00007f9e18a8e000 nid=0x7c97 in Object.wait() [0x00007f9dd846a000]
  6.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  7.     at java.lang.Object.wait(Native Method)
  8.     at org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1014)
  9.     - locked <0x000000019a248e48> (a org.apache.hadoop.ipc.Client$Connection)
  10.     at org.apache.hadoop.ipc.Client$Connection.run(Client.java:1058)
  11.  
  12. "Timer for 'HBase' metrics system" #14624 daemon prio=5 os_prio=0 tid=0x00007f9e36dde800 nid=0x7c16 in Object.wait() [0x00007f9ddf5d6000]
  13.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  14.     at java.lang.Object.wait(Native Method)
  15.     at java.util.TimerThread.mainLoop(Timer.java:552)
  16.     - locked <0x00000001866f6a48> (a java.util.TaskQueue)
  17.     at java.util.TimerThread.run(Timer.java:505)
  18.  
  19. "ResponseProcessor for block BP-1546953902-172.20.113.124-1567252132354:blk_1074278461_538003" #14565 daemon prio=5 os_prio=0 tid=0x00007f9e18fb3000 nid=0x70c8 runnable [0x00007f9ddac92000]
  20.    java.lang.Thread.State: RUNNABLE
  21.     at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  22.     at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
  23.     at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
  24.     at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
  25.     - locked <0x0000000151236f20> (a sun.nio.ch.Util$3)
  26.     - locked <0x0000000151236fc8> (a java.util.Collections$UnmodifiableSet)
  27.     - locked <0x000000014a0eb2e8> (a sun.nio.ch.EPollSelectorImpl)
  28.     at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
  29.     at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335)
  30.     at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157)
  31.     at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161)
  32.     at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131)
  33.     at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118)
  34.     at java.io.FilterInputStream.read(FilterInputStream.java:83)
  35.     at java.io.FilterInputStream.read(FilterInputStream.java:83)
  36.     at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:400)
  37.     at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:213)
  38.     at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1073)
  39.  
  40. "DataStreamer for file /user/hbase/WAL/WALs/ip-172-20-113-118.us-west-2.compute.internal,16020,1568477753544/ip-172-20-113-118.us-west-2.compute.internal%2C16020%2C1568477753544.ip-172-20-113-118.us-west-2.compute.internal%2C16020%2C1568477753544.regiongroup-1.1568646706571 block BP-1546953902-172.20.113.124-1567252132354:blk_1074278461_538003" #14564 daemon prio=5 os_prio=0 tid=0x00007f9e18e87000 nid=0x70c5 in Object.wait() [0x00007f9dd8369000]
  41.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  42.     at java.lang.Object.wait(Native Method)
  43.     at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:672)
  44.     - locked <0x00000001c368bdc0> (a java.util.LinkedList)
  45.  
  46. "ResponseProcessor for block BP-1546953902-172.20.113.124-1567252132354:blk_1074278460_538002" #14563 daemon prio=5 os_prio=0 tid=0x00007f9e37ca9800 nid=0x70c4 runnable [0x00007f9dddcc0000]
  47.    java.lang.Thread.State: RUNNABLE
  48.     at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  49.     at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
  50.     at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
  51.     at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
  52.     - locked <0x00000001b1648500> (a sun.nio.ch.Util$3)
  53.     - locked <0x00000001b16484e8> (a java.util.Collections$UnmodifiableSet)
  54.     - locked <0x0000000216ae9370> (a sun.nio.ch.EPollSelectorImpl)
  55.     at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
  56.     at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335)
  57.     at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157)
  58.     at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161)
  59.     at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131)
  60.     at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118)
  61.     at java.io.FilterInputStream.read(FilterInputStream.java:83)
  62.     at java.io.FilterInputStream.read(FilterInputStream.java:83)
  63.     at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:400)
  64.     at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:213)
  65.     at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1073)
  66.  
  67. "DataStreamer for file /user/hbase/WAL/WALs/ip-172-20-113-118.us-west-2.compute.internal,16020,1568477753544/ip-172-20-113-118.us-west-2.compute.internal%2C16020%2C1568477753544.ip-172-20-113-118.us-west-2.compute.internal%2C16020%2C1568477753544.regiongroup-0.1568646706556 block BP-1546953902-172.20.113.124-1567252132354:blk_1074278460_538002" #14562 daemon prio=5 os_prio=0 tid=0x00007f9e1887c800 nid=0x70c1 in Object.wait() [0x00007f9dddbbf000]
  68.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  69.     at java.lang.Object.wait(Native Method)
  70.     at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:672)
  71.     - locked <0x00000001c368bde0> (a java.util.LinkedList)
  72.  
  73. "ResponseProcessor for block BP-1546953902-172.20.113.124-1567252132354:blk_1074278459_538001" #14561 daemon prio=5 os_prio=0 tid=0x00007f9e19590000 nid=0x70c0 runnable [0x00007f9dd987e000]
  74.    java.lang.Thread.State: RUNNABLE
  75.     at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  76.     at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
  77.     at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
  78.     at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
  79.     - locked <0x00000001e06b23b8> (a sun.nio.ch.Util$3)
  80.     - locked <0x00000001e06b23a0> (a java.util.Collections$UnmodifiableSet)
  81.     - locked <0x000000017ac3bed8> (a sun.nio.ch.EPollSelectorImpl)
  82.     at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
  83.     at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335)
  84.     at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157)
  85.     at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161)
  86.     at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131)
  87.     at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118)
  88.     at java.io.FilterInputStream.read(FilterInputStream.java:83)
  89.     at java.io.FilterInputStream.read(FilterInputStream.java:83)
  90.     at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:400)
  91.     at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:213)
  92.     at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1073)
  93.  
  94. "DataStreamer for file /user/hbase/WAL/WALs/ip-172-20-113-118.us-west-2.compute.internal,16020,1568477753544/ip-172-20-113-118.us-west-2.compute.internal%2C16020%2C1568477753544.ip-172-20-113-118.us-west-2.compute.internal%2C16020%2C1568477753544.regiongroup-2.1568646706541 block BP-1546953902-172.20.113.124-1567252132354:blk_1074278459_538001" #14560 daemon prio=5 os_prio=0 tid=0x00007f9e18220800 nid=0x70bd in Object.wait() [0x00007f9dd967c000]
  95.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  96.     at java.lang.Object.wait(Native Method)
  97.     at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:672)
  98.     - locked <0x00000001c3682b38> (a java.util.LinkedList)
  99.  
  100. "ResponseProcessor for block BP-1546953902-172.20.113.124-1567252132354:blk_1074278458_538000" #14559 daemon prio=5 os_prio=0 tid=0x00007f9e3781c000 nid=0x70bb runnable [0x00007f9dd745a000]
  101.    java.lang.Thread.State: RUNNABLE
  102.     at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  103.     at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
  104.     at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
  105.     at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
  106.     - locked <0x0000000149f0b580> (a sun.nio.ch.Util$3)
  107.     - locked <0x0000000149f0b568> (a java.util.Collections$UnmodifiableSet)
  108.     - locked <0x0000000151053e98> (a sun.nio.ch.EPollSelectorImpl)
  109.     at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
  110.     at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335)
  111.     at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157)
  112.     at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161)
  113.     at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131)
  114.     at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118)
  115.     at java.io.FilterInputStream.read(FilterInputStream.java:83)
  116.     at java.io.FilterInputStream.read(FilterInputStream.java:83)
  117.     at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:400)
  118.     at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:213)
  119.     at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1073)
  120.  
  121. "DataStreamer for file /user/hbase/WAL/WALs/ip-172-20-113-118.us-west-2.compute.internal,16020,1568477753544/ip-172-20-113-118.us-west-2.compute.internal%2C16020%2C1568477753544.ip-172-20-113-118.us-west-2.compute.internal%2C16020%2C1568477753544.regiongroup-3.1568646706520 block BP-1546953902-172.20.113.124-1567252132354:blk_1074278458_538000" #14558 daemon prio=5 os_prio=0 tid=0x00007f9e18a88000 nid=0x70b9 in Object.wait() [0x00007f9dd997f000]
  122.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  123.     at java.lang.Object.wait(Native Method)
  124.     at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:672)
  125.     - locked <0x00000001c3682b58> (a java.util.LinkedList)
  126.  
  127. "IPC Client (1552133633) connection to ip-172-20-113-124.us-west-2.compute.internal/172.20.113.124:16000 from hbase" #7646 daemon prio=5 os_prio=0 tid=0x00007f9e206ef000 nid=0x5476 in Object.wait() [0x00007f9de1ef9000]
  128.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  129.     at java.lang.Object.wait(Native Method)
  130.     at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.waitForWork(BlockingRpcConnection.java:322)
  131.     - locked <0x0000000149e307e0> (a org.apache.hadoop.hbase.ipc.BlockingRpcConnection)
  132.     at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.run(BlockingRpcConnection.java:333)
  133.     at java.lang.Thread.run(Thread.java:748)
  134.  
  135. "phoenix-update-statistics-3" #4304 daemon prio=5 os_prio=0 tid=0x00007f9e20729800 nid=0x7812 waiting on condition [0x00007f9dda58d000]
  136.    java.lang.Thread.State: WAITING (parking)
  137.     at sun.misc.Unsafe.park(Native Method)
  138.     - parking to wait for  <0x000000015bde04e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  139.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  140.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  141.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  142.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  143.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  144.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  145.     at java.lang.Thread.run(Thread.java:748)
  146.  
  147. "phoenix-update-statistics-2" #4023 daemon prio=5 os_prio=0 tid=0x00007f9e37251000 nid=0x5469 waiting on condition [0x00007f9dd7f65000]
  148.    java.lang.Thread.State: WAITING (parking)
  149.     at sun.misc.Unsafe.park(Native Method)
  150.     - parking to wait for  <0x000000015bde04e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  151.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  152.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  153.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  154.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  155.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  156.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  157.     at java.lang.Thread.run(Thread.java:748)
  158.  
  159. "phoenix-update-statistics-1" #3692 daemon prio=5 os_prio=0 tid=0x00007f9e3450f000 nid=0x3ff2 waiting on condition [0x00007f9ddb99f000]
  160.    java.lang.Thread.State: WAITING (parking)
  161.     at sun.misc.Unsafe.park(Native Method)
  162.     - parking to wait for  <0x000000015bde04e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  163.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  164.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  165.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  166.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  167.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  168.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  169.     at java.lang.Thread.run(Thread.java:748)
  170.  
  171. "hfile-prefetch-1568492802864" #3420 daemon prio=5 os_prio=0 tid=0x00007f9e37852000 nid=0x3d0b waiting on condition [0x00007f9ddf6d7000]
  172.    java.lang.Thread.State: WAITING (parking)
  173.     at sun.misc.Unsafe.park(Native Method)
  174.     - parking to wait for  <0x000000015c83c8f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  175.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  176.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  177.     at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)
  178.     at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
  179.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  180.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  181.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  182.     at java.lang.Thread.run(Thread.java:748)
  183.  
  184. "hfile-prefetch-1568492801945" #3411 daemon prio=5 os_prio=0 tid=0x00007f9e2131d000 nid=0x3d02 waiting on condition [0x00007f9de04e1000]
  185.    java.lang.Thread.State: WAITING (parking)
  186.     at sun.misc.Unsafe.park(Native Method)
  187.     - parking to wait for  <0x000000015c83c8f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  188.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  189.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  190.     at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)
  191.     at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
  192.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  193.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  194.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  195.     at java.lang.Thread.run(Thread.java:748)
  196.  
  197. "hfile-prefetch-1568492801177" #3402 daemon prio=5 os_prio=0 tid=0x00007f9e19052800 nid=0x3cf8 waiting for monitor entry [0x00007f9dd644a000]
  198.    java.lang.Thread.State: BLOCKED (on object monitor)
  199.     at com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream.getPos(S3FSInputStream.java:251)
  200.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  201.     at org.apache.hadoop.fs.BufferedFSInputStream.seek(BufferedFSInputStream.java:84)
  202.     at org.apache.hadoop.fs.FSDataInputStream.seek(FSDataInputStream.java:65)
  203.     at org.apache.hadoop.hbase.io.hfile.HFileUtil.seekOnMultipleSources(HFileUtil.java:38)
  204.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1478)
  205.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  206.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  207.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  208.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$1.run(HFileReaderV2.java:209)
  209.     at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
  210.     at java.util.concurrent.FutureTask.run(FutureTask.java:266)
  211.     at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180)
  212.     at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
  213.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  214.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  215.     at java.lang.Thread.run(Thread.java:748)
  216.  
  217. "hfile-prefetch-1568492800342" #3392 daemon prio=5 os_prio=0 tid=0x00007f9e34a6d000 nid=0x3ceb waiting for monitor entry [0x00007f9ddcaae000]
  218.    java.lang.Thread.State: BLOCKED (on object monitor)
  219.     at com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream.getPos(S3FSInputStream.java:251)
  220.     - waiting to lock <0x000000017b396370> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  221.     at org.apache.hadoop.fs.BufferedFSInputStream.seek(BufferedFSInputStream.java:84)
  222.     at org.apache.hadoop.fs.FSDataInputStream.seek(FSDataInputStream.java:65)
  223.     at org.apache.hadoop.hbase.io.hfile.HFileUtil.seekOnMultipleSources(HFileUtil.java:38)
  224.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1478)
  225.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  226.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  227.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  228.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$1.run(HFileReaderV2.java:209)
  229.     at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
  230.     at java.util.concurrent.FutureTask.run(FutureTask.java:266)
  231.     at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180)
  232.     at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
  233.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  234.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  235.     at java.lang.Thread.run(Thread.java:748)
  236.  
  237. "phoenix-update-statistics-0" #2839 daemon prio=5 os_prio=0 tid=0x00007f9e37ffe800 nid=0x905 waiting on condition [0x00007f9dd775d000]
  238.    java.lang.Thread.State: WAITING (parking)
  239.     at sun.misc.Unsafe.park(Native Method)
  240.     - parking to wait for  <0x000000015bde04e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  241.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  242.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  243.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  244.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  245.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  246.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  247.     at java.lang.Thread.run(Thread.java:748)
  248.  
  249. "ip-172-20-113-118.us-west-2.compute.internal,16020,1568477753544_ChoreService_3" #2835 daemon prio=5 os_prio=0 tid=0x00007f9e203f8800 nid=0x893 waiting on condition [0x00007f9dd5e44000]
  250.    java.lang.Thread.State: WAITING (parking)
  251.     at sun.misc.Unsafe.park(Native Method)
  252.     - parking to wait for  <0x0000000150f213d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  253.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  254.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  255.     at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)
  256.     at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
  257.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  258.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  259.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  260.     at java.lang.Thread.run(Thread.java:748)
  261.  
  262. "ip-172-20-113-118.us-west-2.compute.internal,16020,1568477753544_ChoreService_2" #2834 daemon prio=5 os_prio=0 tid=0x00007f9e18fea000 nid=0x892 waiting on condition [0x00007f9ddbda1000]
  263.    java.lang.Thread.State: TIMED_WAITING (parking)
  264.     at sun.misc.Unsafe.park(Native Method)
  265.     - parking to wait for  <0x0000000150f213d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  266.     at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
  267.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
  268.     at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
  269.     at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
  270.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  271.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  272.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  273.     at java.lang.Thread.run(Thread.java:748)
  274.  
  275. "regionserver/ip-172-20-113-118.us-west-2.compute.internal/172.20.113.118:16020-shortCompactions-1568478068632-EventThread" #2830 daemon prio=5 os_prio=0 tid=0x00007f9e36154000 nid=0x87a waiting on condition [0x00007f9ddccb0000]
  276.    java.lang.Thread.State: WAITING (parking)
  277.     at sun.misc.Unsafe.park(Native Method)
  278.     - parking to wait for  <0x0000000150fc4a78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  279.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  280.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  281.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  282.     at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:504)
  283.  
  284. "regionserver/ip-172-20-113-118.us-west-2.compute.internal/172.20.113.118:16020-shortCompactions-1568478068632-SendThread(ip-172-20-113-124.us-west-2.compute.internal:2181)" #2829 daemon prio=5 os_prio=0 tid=0x00007f9e36153800 nid=0x879 runnable [0x00007f9dda389000]
  285.    java.lang.Thread.State: RUNNABLE
  286.     at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  287.     at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
  288.     at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
  289.     at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
  290.     - locked <0x00000001a76b7440> (a sun.nio.ch.Util$3)
  291.     - locked <0x00000001a76b7428> (a java.util.Collections$UnmodifiableSet)
  292.     - locked <0x000000018cda9e80> (a sun.nio.ch.EPollSelectorImpl)
  293.     at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
  294.     at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)
  295.     at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1145)
  296.  
  297. "PHOENIX-SCANNER-RENEW-LEASE-thread-10" #2827 daemon prio=5 os_prio=0 tid=0x0000000002adb000 nid=0x877 waiting on condition [0x00007f9dd785e000]
  298.    java.lang.Thread.State: WAITING (parking)
  299.     at sun.misc.Unsafe.park(Native Method)
  300.     - parking to wait for  <0x0000000149a02b50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  301.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  302.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  303.     at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)
  304.     at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
  305.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  306.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  307.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  308.     at java.lang.Thread.run(Thread.java:748)
  309.  
  310. "PHOENIX-SCANNER-RENEW-LEASE-thread-5" #2823 daemon prio=5 os_prio=0 tid=0x00007f9e2106d000 nid=0x876 waiting on condition [0x00007f9dd8268000]
  311.    java.lang.Thread.State: WAITING (parking)
  312.     at sun.misc.Unsafe.park(Native Method)
  313.     - parking to wait for  <0x0000000149a02b50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  314.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  315.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  316.     at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)
  317.     at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
  318.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  319.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  320.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  321.     at java.lang.Thread.run(Thread.java:748)
  322.  
  323. "PHOENIX-SCANNER-RENEW-LEASE-thread-9" #2826 daemon prio=5 os_prio=0 tid=0x00007f9e3614f800 nid=0x874 waiting on condition [0x00007f9dd8c72000]
  324.    java.lang.Thread.State: WAITING (parking)
  325.     at sun.misc.Unsafe.park(Native Method)
  326.     - parking to wait for  <0x0000000149a02b50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  327.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  328.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  329.     at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)
  330.     at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
  331.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  332.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  333.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  334.     at java.lang.Thread.run(Thread.java:748)
  335.  
  336. "PHOENIX-SCANNER-RENEW-LEASE-thread-7" #2825 daemon prio=5 os_prio=0 tid=0x00007f9dd0047000 nid=0x873 waiting on condition [0x00007f9ddaa90000]
  337.    java.lang.Thread.State: WAITING (parking)
  338.     at sun.misc.Unsafe.park(Native Method)
  339.     - parking to wait for  <0x0000000149a02b50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  340.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  341.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  342.     at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)
  343.     at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
  344.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  345.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  346.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  347.     at java.lang.Thread.run(Thread.java:748)
  348.  
  349. "PHOENIX-SCANNER-RENEW-LEASE-thread-8" #2824 daemon prio=5 os_prio=0 tid=0x00007f9e185d1000 nid=0x872 waiting on condition [0x00007f9dd684e000]
  350.    java.lang.Thread.State: TIMED_WAITING (parking)
  351.     at sun.misc.Unsafe.park(Native Method)
  352.     - parking to wait for  <0x0000000149a02b50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  353.     at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
  354.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
  355.     at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
  356.     at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
  357.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  358.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  359.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  360.     at java.lang.Thread.run(Thread.java:748)
  361.  
  362. "PHOENIX-SCANNER-RENEW-LEASE-thread-6" #2822 daemon prio=5 os_prio=0 tid=0x00007f9dd0046000 nid=0x871 waiting on condition [0x00007f9dda48a000]
  363.    java.lang.Thread.State: WAITING (parking)
  364.     at sun.misc.Unsafe.park(Native Method)
  365.     - parking to wait for  <0x0000000149a02b50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  366.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  367.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  368.     at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)
  369.     at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
  370.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  371.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  372.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  373.     at java.lang.Thread.run(Thread.java:748)
  374.  
  375. "PHOENIX-SCANNER-RENEW-LEASE-thread-4" #2821 daemon prio=5 os_prio=0 tid=0x00007f9e185d0800 nid=0x870 waiting on condition [0x00007f9ddae98000]
  376.    java.lang.Thread.State: WAITING (parking)
  377.     at sun.misc.Unsafe.park(Native Method)
  378.     - parking to wait for  <0x0000000149a02b50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  379.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  380.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  381.     at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)
  382.     at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
  383.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  384.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  385.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  386.     at java.lang.Thread.run(Thread.java:748)
  387.  
  388. "PHOENIX-SCANNER-RENEW-LEASE-thread-3" #2820 daemon prio=5 os_prio=0 tid=0x00007f9dd0044800 nid=0x86f waiting on condition [0x00007f9ddbea2000]
  389.    java.lang.Thread.State: WAITING (parking)
  390.     at sun.misc.Unsafe.park(Native Method)
  391.     - parking to wait for  <0x0000000149a02b50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  392.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  393.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  394.     at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)
  395.     at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
  396.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  397.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  398.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  399.     at java.lang.Thread.run(Thread.java:748)
  400.  
  401. "PHOENIX-SCANNER-RENEW-LEASE-thread-2" #2819 daemon prio=5 os_prio=0 tid=0x00007f9dd0044000 nid=0x86e waiting on condition [0x00007f9dd7b61000]
  402.    java.lang.Thread.State: WAITING (parking)
  403.     at sun.misc.Unsafe.park(Native Method)
  404.     - parking to wait for  <0x0000000149a02b50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  405.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  406.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  407.     at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)
  408.     at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
  409.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  410.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  411.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  412.     at java.lang.Thread.run(Thread.java:748)
  413.  
  414. "PHOENIX-SCANNER-RENEW-LEASE-thread-1" #2818 daemon prio=5 os_prio=0 tid=0x00007f9dd0041800 nid=0x86d waiting on condition [0x00007f9dd856b000]
  415.    java.lang.Thread.State: WAITING (parking)
  416.     at sun.misc.Unsafe.park(Native Method)
  417.     - parking to wait for  <0x0000000149a02b50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  418.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  419.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  420.     at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)
  421.     at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
  422.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  423.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  424.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  425.     at java.lang.Thread.run(Thread.java:748)
  426.  
  427. "regionserver/ip-172-20-113-118.us-west-2.compute.internal/172.20.113.118:16020-shortCompactions-1568478068632-EventThread" #2803 daemon prio=5 os_prio=0 tid=0x00007f9e37a05000 nid=0x85d waiting on condition [0x00007f9dddfc3000]
  428.    java.lang.Thread.State: WAITING (parking)
  429.     at sun.misc.Unsafe.park(Native Method)
  430.     - parking to wait for  <0x00000001a6452818> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  431.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  432.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  433.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  434.     at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:504)
  435.  
  436. "regionserver/ip-172-20-113-118.us-west-2.compute.internal/172.20.113.118:16020-shortCompactions-1568478068632-SendThread(ip-172-20-113-124.us-west-2.compute.internal:2181)" #2802 daemon prio=5 os_prio=0 tid=0x00007f9e379ea800 nid=0x85c runnable [0x00007f9dda98f000]
  437.    java.lang.Thread.State: RUNNABLE
  438.     at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  439.     at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
  440.     at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
  441.     at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
  442.     - locked <0x00000001a6451138> (a sun.nio.ch.Util$3)
  443.     - locked <0x00000001a6451120> (a java.util.Collections$UnmodifiableSet)
  444.     - locked <0x000000017ff202f0> (a sun.nio.ch.EPollSelectorImpl)
  445.     at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
  446.     at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)
  447.     at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1145)
  448.  
  449. "RS_LOG_REPLAY_OPS-ip-172-20-113-118:16020-1" #1396 prio=5 os_prio=0 tid=0x00007f9e183aa000 nid=0x5bca waiting on condition [0x00007f9ddad93000]
  450.    java.lang.Thread.State: WAITING (parking)
  451.     at sun.misc.Unsafe.park(Native Method)
  452.     - parking to wait for  <0x0000000151561130> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  453.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  454.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  455.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  456.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  457.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  458.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  459.     at java.lang.Thread.run(Thread.java:748)
  460.  
  461. "RS_CLOSE_REGION-ip-172-20-113-118:16020-2" #1103 prio=5 os_prio=0 tid=0x00000000020da000 nid=0x3d6a waiting on condition [0x00007f9ddb89e000]
  462.    java.lang.Thread.State: WAITING (parking)
  463.     at sun.misc.Unsafe.park(Native Method)
  464.     - parking to wait for  <0x0000000151561400> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  465.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  466.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  467.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  468.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  469.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  470.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  471.     at java.lang.Thread.run(Thread.java:748)
  472.  
  473. "RS_CLOSE_REGION-ip-172-20-113-118:16020-1" #930 prio=5 os_prio=0 tid=0x000000000220f800 nid=0x36da waiting on condition [0x00007f9dd8e74000]
  474.    java.lang.Thread.State: WAITING (parking)
  475.     at sun.misc.Unsafe.park(Native Method)
  476.     - parking to wait for  <0x0000000151561400> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  477.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  478.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  479.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  480.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  481.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  482.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  483.     at java.lang.Thread.run(Thread.java:748)
  484.  
  485. "RS_CLOSE_REGION-ip-172-20-113-118:16020-0" #768 prio=5 os_prio=0 tid=0x00000000022d3800 nid=0x2cb3 waiting on condition [0x00007f9dd977d000]
  486.    java.lang.Thread.State: WAITING (parking)
  487.     at sun.misc.Unsafe.park(Native Method)
  488.     - parking to wait for  <0x0000000151561400> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  489.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  490.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  491.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  492.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  493.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  494.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  495.     at java.lang.Thread.run(Thread.java:748)
  496.  
  497. "RS_COMPACTED_FILES_DISCHARGER-ip-172-20-113-118:16020-9" #652 prio=5 os_prio=0 tid=0x00007f9e1864a800 nid=0x2b37 waiting on condition [0x00007f9ddaf99000]
  498.    java.lang.Thread.State: WAITING (parking)
  499.     at sun.misc.Unsafe.park(Native Method)
  500.     - parking to wait for  <0x0000000151560fc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  501.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  502.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  503.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  504.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  505.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  506.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  507.     at java.lang.Thread.run(Thread.java:748)
  508.  
  509. "RS_COMPACTED_FILES_DISCHARGER-ip-172-20-113-118:16020-8" #651 prio=5 os_prio=0 tid=0x00007f9e18649800 nid=0x2b36 waiting on condition [0x00007f9ddb09a000]
  510.    java.lang.Thread.State: WAITING (parking)
  511.     at sun.misc.Unsafe.park(Native Method)
  512.     - parking to wait for  <0x0000000151560fc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  513.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  514.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  515.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  516.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  517.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  518.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  519.     at java.lang.Thread.run(Thread.java:748)
  520.  
  521. "RS_COMPACTED_FILES_DISCHARGER-ip-172-20-113-118:16020-7" #650 prio=5 os_prio=0 tid=0x00007f9e18649000 nid=0x2b35 waiting on condition [0x00007f9ddb19b000]
  522.    java.lang.Thread.State: WAITING (parking)
  523.     at sun.misc.Unsafe.park(Native Method)
  524.     - parking to wait for  <0x0000000151560fc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  525.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  526.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  527.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  528.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  529.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  530.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  531.     at java.lang.Thread.run(Thread.java:748)
  532.  
  533. "RS_COMPACTED_FILES_DISCHARGER-ip-172-20-113-118:16020-6" #649 prio=5 os_prio=0 tid=0x00007f9e185fa000 nid=0x2b34 waiting on condition [0x00007f9dda086000]
  534.    java.lang.Thread.State: WAITING (parking)
  535.     at sun.misc.Unsafe.park(Native Method)
  536.     - parking to wait for  <0x0000000151560fc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  537.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  538.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  539.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  540.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  541.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  542.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  543.     at java.lang.Thread.run(Thread.java:748)
  544.  
  545. "RS_COMPACTED_FILES_DISCHARGER-ip-172-20-113-118:16020-5" #648 prio=5 os_prio=0 tid=0x00007f9e185f9000 nid=0x2b33 waiting on condition [0x00007f9dd9076000]
  546.    java.lang.Thread.State: WAITING (parking)
  547.     at sun.misc.Unsafe.park(Native Method)
  548.     - parking to wait for  <0x0000000151560fc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  549.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  550.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  551.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  552.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  553.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  554.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  555.     at java.lang.Thread.run(Thread.java:748)
  556.  
  557. "RS_COMPACTED_FILES_DISCHARGER-ip-172-20-113-118:16020-4" #647 prio=5 os_prio=0 tid=0x00007f9e185f8800 nid=0x2b32 waiting on condition [0x00007f9dda88e000]
  558.    java.lang.Thread.State: WAITING (parking)
  559.     at sun.misc.Unsafe.park(Native Method)
  560.     - parking to wait for  <0x0000000151560fc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  561.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  562.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  563.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  564.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  565.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  566.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  567.     at java.lang.Thread.run(Thread.java:748)
  568.  
  569. "RS_COMPACTED_FILES_DISCHARGER-ip-172-20-113-118:16020-3" #646 prio=5 os_prio=0 tid=0x00007f9e185f7800 nid=0x2b31 waiting on condition [0x00007f9dd9f85000]
  570.    java.lang.Thread.State: WAITING (parking)
  571.     at sun.misc.Unsafe.park(Native Method)
  572.     - parking to wait for  <0x0000000151560fc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  573.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  574.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  575.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  576.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  577.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  578.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  579.     at java.lang.Thread.run(Thread.java:748)
  580.  
  581. "RS_COMPACTED_FILES_DISCHARGER-ip-172-20-113-118:16020-2" #645 prio=5 os_prio=0 tid=0x00007f9e185f7000 nid=0x2b30 waiting on condition [0x00007f9dd8b71000]
  582.    java.lang.Thread.State: WAITING (parking)
  583.     at sun.misc.Unsafe.park(Native Method)
  584.     - parking to wait for  <0x0000000151560fc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  585.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  586.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  587.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  588.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  589.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  590.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  591.     at java.lang.Thread.run(Thread.java:748)
  592.  
  593. "RS_COMPACTED_FILES_DISCHARGER-ip-172-20-113-118:16020-1" #644 prio=5 os_prio=0 tid=0x00007f9e185f6000 nid=0x2b2f waiting on condition [0x00007f9dd654b000]
  594.    java.lang.Thread.State: WAITING (parking)
  595.     at sun.misc.Unsafe.park(Native Method)
  596.     - parking to wait for  <0x0000000151560fc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  597.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  598.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  599.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  600.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  601.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  602.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  603.     at java.lang.Thread.run(Thread.java:748)
  604.  
  605. "RS_COMPACTED_FILES_DISCHARGER-ip-172-20-113-118:16020-0" #643 prio=5 os_prio=0 tid=0x00007f9e1892b800 nid=0x2b2e waiting on condition [0x00007f9de06e3000]
  606.    java.lang.Thread.State: WAITING (parking)
  607.     at sun.misc.Unsafe.park(Native Method)
  608.     - parking to wait for  <0x0000000151560fc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  609.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  610.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  611.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  612.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  613.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  614.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  615.     at java.lang.Thread.run(Thread.java:748)
  616.  
  617. "regionserver/ip-172-20-113-118.us-west-2.compute.internal/172.20.113.118:16020-shortCompactions-1568478085425" #586 daemon prio=5 os_prio=0 tid=0x00007f9e181ae800 nid=0x2a84 waiting on condition [0x00007f9dde2c4000]
  618.    java.lang.Thread.State: WAITING (parking)
  619.     at sun.misc.Unsafe.park(Native Method)
  620.     - parking to wait for  <0x0000000151561970> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  621.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  622.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  623.     at java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:550)
  624.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  625.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  626.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  627.     at java.lang.Thread.run(Thread.java:748)
  628.  
  629. "regionserver/ip-172-20-113-118.us-west-2.compute.internal/172.20.113.118:16020-shortCompactions-1568478077853" #552 daemon prio=5 os_prio=0 tid=0x00007f9e3785a800 nid=0x2a4f waiting on condition [0x00007f9dd7056000]
  630.    java.lang.Thread.State: WAITING (parking)
  631.     at sun.misc.Unsafe.park(Native Method)
  632.     - parking to wait for  <0x0000000151561970> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  633.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  634.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  635.     at java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:550)
  636.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  637.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  638.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  639.     at java.lang.Thread.run(Thread.java:748)
  640.  
  641. "regionserver/ip-172-20-113-118.us-west-2.compute.internal/172.20.113.118:16020-shortCompactions-1568478068632" #491 daemon prio=5 os_prio=0 tid=0x00007f9e20b20000 nid=0x2a09 runnable [0x00007f9dd6a4e000]
  642.    java.lang.Thread.State: RUNNABLE
  643.     at java.net.SocketInputStream.socketRead0(Native Method)
  644.     at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
  645.     at java.net.SocketInputStream.read(SocketInputStream.java:171)
  646.     at java.net.SocketInputStream.read(SocketInputStream.java:141)
  647.     at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
  648.     at sun.security.ssl.InputRecord.read(InputRecord.java:503)
  649.     at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:975)
  650.     - locked <0x000000012f2b7800> (a java.lang.Object)
  651.     at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:933)
  652.     at sun.security.ssl.AppInputStream.read(AppInputStream.java:105)
  653.     - locked <0x000000012f2b80d0> (a sun.security.ssl.AppInputStream)
  654.     at com.amazon.ws.emr.hadoop.fs.shaded.org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:139)
  655.     at com.amazon.ws.emr.hadoop.fs.shaded.org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:155)
  656.     at com.amazon.ws.emr.hadoop.fs.shaded.org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:284)
  657.     at com.amazon.ws.emr.hadoop.fs.shaded.org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
  658.     at com.amazon.ws.emr.hadoop.fs.shaded.org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
  659.     at com.amazon.ws.emr.hadoop.fs.shaded.org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
  660.     at com.amazon.ws.emr.hadoop.fs.shaded.org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
  661.     at com.amazon.ws.emr.hadoop.fs.shaded.org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
  662.     at com.amazon.ws.emr.hadoop.fs.shaded.org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
  663.     at com.amazon.ws.emr.hadoop.fs.shaded.com.amazonaws.http.protocol.SdkHttpRequestExecutor.doReceiveResponse(SdkHttpRequestExecutor.java:82)
  664.     at com.amazon.ws.emr.hadoop.fs.shaded.org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
  665.     at com.amazon.ws.emr.hadoop.fs.shaded.org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
  666.     at com.amazon.ws.emr.hadoop.fs.shaded.org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
  667.     at com.amazon.ws.emr.hadoop.fs.shaded.org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
  668.     at com.amazon.ws.emr.hadoop.fs.shaded.org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
  669.     at com.amazon.ws.emr.hadoop.fs.shaded.org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
  670.     at com.amazon.ws.emr.hadoop.fs.shaded.com.amazonaws.http.apache.client.impl.SdkHttpClient.execute(SdkHttpClient.java:72)
  671.     at com.amazon.ws.emr.hadoop.fs.shaded.com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1256)
  672.     at com.amazon.ws.emr.hadoop.fs.shaded.com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1072)
  673.     at com.amazon.ws.emr.hadoop.fs.shaded.com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:745)
  674.     at com.amazon.ws.emr.hadoop.fs.shaded.com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:719)
  675.     at com.amazon.ws.emr.hadoop.fs.shaded.com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:701)
  676.     at com.amazon.ws.emr.hadoop.fs.shaded.com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:669)
  677.     at com.amazon.ws.emr.hadoop.fs.shaded.com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:651)
  678.     at com.amazon.ws.emr.hadoop.fs.shaded.com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:515)
  679.     at com.amazon.ws.emr.hadoop.fs.shaded.com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:4443)
  680.     at com.amazon.ws.emr.hadoop.fs.shaded.com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:4390)
  681.     at com.amazon.ws.emr.hadoop.fs.shaded.com.amazonaws.services.s3.AmazonS3Client.getObjectMetadata(AmazonS3Client.java:1280)
  682.     at com.amazon.ws.emr.hadoop.fs.s3.lite.call.GetObjectMetadataCall.perform(GetObjectMetadataCall.java:22)
  683.     at com.amazon.ws.emr.hadoop.fs.s3.lite.call.GetObjectMetadataCall.perform(GetObjectMetadataCall.java:8)
  684.     at com.amazon.ws.emr.hadoop.fs.s3.lite.executor.GlobalS3Executor.execute(GlobalS3Executor.java:91)
  685.     at com.amazon.ws.emr.hadoop.fs.s3.lite.AmazonS3LiteClient.invoke(AmazonS3LiteClient.java:184)
  686.     at com.amazon.ws.emr.hadoop.fs.s3.lite.AmazonS3LiteClient.getObjectMetadata(AmazonS3LiteClient.java:96)
  687.     at com.amazon.ws.emr.hadoop.fs.s3.lite.AbstractAmazonS3Lite.getObjectMetadata(AbstractAmazonS3Lite.java:38)
  688.     at com.amazon.ws.emr.hadoop.fs.cse.CSEUtils.getPlaintextLength(CSEUtils.java:79)
  689.     at com.amazon.ws.emr.hadoop.fs.cse.CSEUtils.getPlaintextLength(CSEUtils.java:72)
  690.     at com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream.open(S3FSInputStream.java:66)
  691.     at com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream.retrieveInputStreamWithInfo(S3FSInputStream.java:192)
  692.     at com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream.reopenStream(S3FSInputStream.java:180)
  693.     - eliminated <0x000000017b396370> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  694.     at com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream.seekStream(S3FSInputStream.java:246)
  695.     - eliminated <0x000000017b396370> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  696.     at com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream.read(S3FSInputStream.java:117)
  697.     - locked <0x000000017b396370> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  698.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:75)
  699.     - locked <0x000000017b396370> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  700.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  701.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  702.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  703.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  704.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  705.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  706.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  707.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.readNextDataBlock(HFileReaderV2.java:731)
  708.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$EncodedScannerV2.next(HFileReaderV2.java:1288)
  709.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.next(StoreFileScanner.java:181)
  710.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:113)
  711.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:628)
  712.     at org.apache.hadoop.hbase.regionserver.compactions.Compactor.performCompaction(Compactor.java:448)
  713.     at org.apache.hadoop.hbase.regionserver.compactions.Compactor.compact(Compactor.java:323)
  714.     at org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor.compact(DefaultCompactor.java:66)
  715.     at org.apache.hadoop.hbase.regionserver.DefaultStoreEngine$DefaultCompactionContext.compact(DefaultStoreEngine.java:115)
  716.     at org.apache.hadoop.hbase.regionserver.HStore.compact(HStore.java:1350)
  717.     at org.apache.hadoop.hbase.regionserver.HRegion.compact(HRegion.java:2104)
  718.     at org.apache.hadoop.hbase.regionserver.CompactSplitThread$CompactionRunner.doCompaction(CompactSplitThread.java:535)
  719.     at org.apache.hadoop.hbase.regionserver.CompactSplitThread$CompactionRunner.run(CompactSplitThread.java:575)
  720.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  721.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  722.     at java.lang.Thread.run(Thread.java:748)
  723.  
  724. "regionserver/ip-172-20-113-118.us-west-2.compute.internal/172.20.113.118:16020-shortCompactions-1568478066437" #481 daemon prio=5 os_prio=0 tid=0x00000000022f5000 nid=0x29fb waiting on condition [0x00007f9ddb79d000]
  725.    java.lang.Thread.State: WAITING (parking)
  726.     at sun.misc.Unsafe.park(Native Method)
  727.     - parking to wait for  <0x0000000151561970> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  728.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  729.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  730.     at java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:550)
  731.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  732.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  733.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  734.     at java.lang.Thread.run(Thread.java:748)
  735.  
  736. "org.apache.hadoop.hdfs.PeerCache@81f76c5" #332 daemon prio=5 os_prio=0 tid=0x00007f9e185fe000 nid=0x28cb waiting on condition [0x00007f9ddb69c000]
  737.    java.lang.Thread.State: TIMED_WAITING (sleeping)
  738.     at java.lang.Thread.sleep(Native Method)
  739.     at org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253)
  740.     at org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46)
  741.     at org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124)
  742.     at java.lang.Thread.run(Thread.java:748)
  743.  
  744. "RS_LOG_REPLAY_OPS-ip-172-20-113-118:16020-0" #328 prio=5 os_prio=0 tid=0x00007f9e18529000 nid=0x2824 waiting on condition [0x00007f9ddbca0000]
  745.    java.lang.Thread.State: WAITING (parking)
  746.     at sun.misc.Unsafe.park(Native Method)
  747.     - parking to wait for  <0x0000000151561130> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  748.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  749.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  750.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  751.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  752.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  753.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  754.     at java.lang.Thread.run(Thread.java:748)
  755.  
  756. "s3n-worker-19" #317 daemon prio=5 os_prio=0 tid=0x00007f9e18330000 nid=0x27ff waiting on condition [0x00007f9ddbfa3000]
  757.    java.lang.Thread.State: WAITING (parking)
  758.     at sun.misc.Unsafe.park(Native Method)
  759.     - parking to wait for  <0x000000014ffa8498> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  760.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  761.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  762.     at java.util.concurrent.LinkedBlockingDeque.takeFirst(LinkedBlockingDeque.java:492)
  763.     at java.util.concurrent.LinkedBlockingDeque.take(LinkedBlockingDeque.java:680)
  764.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  765.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  766.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  767.     at java.lang.Thread.run(Thread.java:748)
  768.  
  769. "s3n-worker-18" #313 daemon prio=5 os_prio=0 tid=0x0000000002076000 nid=0x27fa waiting on condition [0x00007f9ddc0a4000]
  770.    java.lang.Thread.State: WAITING (parking)
  771.     at sun.misc.Unsafe.park(Native Method)
  772.     - parking to wait for  <0x000000014ffa8498> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  773.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  774.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  775.     at java.util.concurrent.LinkedBlockingDeque.takeFirst(LinkedBlockingDeque.java:492)
  776.     at java.util.concurrent.LinkedBlockingDeque.take(LinkedBlockingDeque.java:680)
  777.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  778.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  779.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  780.     at java.lang.Thread.run(Thread.java:748)
  781.  
  782. "s3n-worker-17" #312 daemon prio=5 os_prio=0 tid=0x0000000001b93000 nid=0x27f9 waiting on condition [0x00007f9ddc3a7000]
  783.    java.lang.Thread.State: WAITING (parking)
  784.     at sun.misc.Unsafe.park(Native Method)
  785.     - parking to wait for  <0x000000014ffa8498> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  786.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  787.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  788.     at java.util.concurrent.LinkedBlockingDeque.takeFirst(LinkedBlockingDeque.java:492)
  789.     at java.util.concurrent.LinkedBlockingDeque.take(LinkedBlockingDeque.java:680)
  790.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  791.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  792.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  793.     at java.lang.Thread.run(Thread.java:748)
  794.  
  795. "s3n-worker-16" #310 daemon prio=5 os_prio=0 tid=0x00007f9e1811a000 nid=0x27f7 waiting on condition [0x00007f9ddc1a5000]
  796.    java.lang.Thread.State: WAITING (parking)
  797.     at sun.misc.Unsafe.park(Native Method)
  798.     - parking to wait for  <0x000000014ffa8498> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  799.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  800.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  801.     at java.util.concurrent.LinkedBlockingDeque.takeFirst(LinkedBlockingDeque.java:492)
  802.     at java.util.concurrent.LinkedBlockingDeque.take(LinkedBlockingDeque.java:680)
  803.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  804.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  805.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  806.     at java.lang.Thread.run(Thread.java:748)
  807.  
  808. "s3n-worker-15" #309 daemon prio=5 os_prio=0 tid=0x00007f9e18118000 nid=0x27f6 waiting on condition [0x00007f9ddc2a6000]
  809.    java.lang.Thread.State: WAITING (parking)
  810.     at sun.misc.Unsafe.park(Native Method)
  811.     - parking to wait for  <0x000000014ffa8498> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  812.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  813.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  814.     at java.util.concurrent.LinkedBlockingDeque.takeFirst(LinkedBlockingDeque.java:492)
  815.     at java.util.concurrent.LinkedBlockingDeque.take(LinkedBlockingDeque.java:680)
  816.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  817.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  818.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  819.     at java.lang.Thread.run(Thread.java:748)
  820.  
  821. "sync.4" #306 prio=5 os_prio=0 tid=0x00007f9e36cf7800 nid=0x27f4 waiting on condition [0x00007f9ddc4a8000]
  822.    java.lang.Thread.State: WAITING (parking)
  823.     at sun.misc.Unsafe.park(Native Method)
  824.     - parking to wait for  <0x0000000151e4fc58> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  825.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  826.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  827.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  828.     at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:1263)
  829.     at java.lang.Thread.run(Thread.java:748)
  830.  
  831. "sync.3" #305 prio=5 os_prio=0 tid=0x00007f9e36cf5800 nid=0x27f3 waiting on condition [0x00007f9ddc5a9000]
  832.    java.lang.Thread.State: WAITING (parking)
  833.     at sun.misc.Unsafe.park(Native Method)
  834.     - parking to wait for  <0x000000015381e098> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  835.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  836.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  837.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  838.     at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:1263)
  839.     at java.lang.Thread.run(Thread.java:748)
  840.  
  841. "sync.2" #304 prio=5 os_prio=0 tid=0x00007f9e36cf4000 nid=0x27f2 waiting on condition [0x00007f9ddc6aa000]
  842.    java.lang.Thread.State: WAITING (parking)
  843.     at sun.misc.Unsafe.park(Native Method)
  844.     - parking to wait for  <0x000000015386e1e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  845.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  846.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  847.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  848.     at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:1263)
  849.     at java.lang.Thread.run(Thread.java:748)
  850.  
  851. "sync.1" #303 prio=5 os_prio=0 tid=0x00007f9e36c7d000 nid=0x27f1 waiting on condition [0x00007f9ddc7ab000]
  852.    java.lang.Thread.State: WAITING (parking)
  853.     at sun.misc.Unsafe.park(Native Method)
  854.     - parking to wait for  <0x0000000151e4fbe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  855.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  856.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  857.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  858.     at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:1263)
  859.     at java.lang.Thread.run(Thread.java:748)
  860.  
  861. "sync.0" #302 prio=5 os_prio=0 tid=0x00007f9e36c7b800 nid=0x27f0 waiting on condition [0x00007f9ddc8ac000]
  862.    java.lang.Thread.State: WAITING (parking)
  863.     at sun.misc.Unsafe.park(Native Method)
  864.     - parking to wait for  <0x0000000151e4fbc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  865.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  866.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  867.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  868.     at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:1263)
  869.     at java.lang.Thread.run(Thread.java:748)
  870.  
  871. "RS_OPEN_REGION-ip-172-20-113-118:16020-2.append-pool8-t1" #307 prio=5 os_prio=0 tid=0x0000000001b34800 nid=0x27ef waiting on condition [0x00007f9ddc9ad000]
  872.    java.lang.Thread.State: WAITING (parking)
  873.     at sun.misc.Unsafe.park(Native Method)
  874.     - parking to wait for  <0x0000000151e4fb50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  875.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  876.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  877.     at com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:45)
  878.     at com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:55)
  879.     at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:123)
  880.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  881.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  882.     at java.lang.Thread.run(Thread.java:748)
  883.  
  884. "RS_OPEN_REGION-ip-172-20-113-118:16020-2" #299 prio=5 os_prio=0 tid=0x00007f9e18331000 nid=0x27ea waiting on condition [0x00007f9ddcdb1000]
  885.    java.lang.Thread.State: WAITING (parking)
  886.     at sun.misc.Unsafe.park(Native Method)
  887.     - parking to wait for  <0x0000000152c918e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  888.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  889.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  890.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  891.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  892.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  893.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  894.     at java.lang.Thread.run(Thread.java:748)
  895.  
  896. "s3n-worker-14" #298 daemon prio=5 os_prio=0 tid=0x00007f9e18003000 nid=0x27e9 waiting on condition [0x00007f9defeb0000]
  897.    java.lang.Thread.State: WAITING (parking)
  898.     at sun.misc.Unsafe.park(Native Method)
  899.     - parking to wait for  <0x000000014ffa8498> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  900.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  901.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  902.     at java.util.concurrent.LinkedBlockingDeque.takeFirst(LinkedBlockingDeque.java:492)
  903.     at java.util.concurrent.LinkedBlockingDeque.take(LinkedBlockingDeque.java:680)
  904.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  905.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  906.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  907.     at java.lang.Thread.run(Thread.java:748)
  908.  
  909. "s3n-worker-13" #293 daemon prio=5 os_prio=0 tid=0x00007f9e37638000 nid=0x27df waiting on condition [0x00007f9ddceb2000]
  910.    java.lang.Thread.State: WAITING (parking)
  911.     at sun.misc.Unsafe.park(Native Method)
  912.     - parking to wait for  <0x000000014ffa8498> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  913.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  914.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  915.     at java.util.concurrent.LinkedBlockingDeque.takeFirst(LinkedBlockingDeque.java:492)
  916.     at java.util.concurrent.LinkedBlockingDeque.take(LinkedBlockingDeque.java:680)
  917.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  918.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  919.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  920.     at java.lang.Thread.run(Thread.java:748)
  921.  
  922. "s3n-worker-12" #292 daemon prio=5 os_prio=0 tid=0x00007f9e37636800 nid=0x27de waiting on condition [0x00007f9ddcfb3000]
  923.    java.lang.Thread.State: WAITING (parking)
  924.     at sun.misc.Unsafe.park(Native Method)
  925.     - parking to wait for  <0x000000014ffa8498> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  926.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  927.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  928.     at java.util.concurrent.LinkedBlockingDeque.takeFirst(LinkedBlockingDeque.java:492)
  929.     at java.util.concurrent.LinkedBlockingDeque.take(LinkedBlockingDeque.java:680)
  930.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  931.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  932.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  933.     at java.lang.Thread.run(Thread.java:748)
  934.  
  935. "s3n-worker-11" #291 daemon prio=5 os_prio=0 tid=0x00007f9e37680000 nid=0x27dd waiting on condition [0x00007f9ddd4b8000]
  936.    java.lang.Thread.State: WAITING (parking)
  937.     at sun.misc.Unsafe.park(Native Method)
  938.     - parking to wait for  <0x000000014ffa8498> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  939.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  940.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  941.     at java.util.concurrent.LinkedBlockingDeque.takeFirst(LinkedBlockingDeque.java:492)
  942.     at java.util.concurrent.LinkedBlockingDeque.take(LinkedBlockingDeque.java:680)
  943.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  944.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  945.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  946.     at java.lang.Thread.run(Thread.java:748)
  947.  
  948. "s3n-worker-10" #289 daemon prio=5 os_prio=0 tid=0x00007f9e1858f000 nid=0x27d8 waiting on condition [0x00007f9ddd0b4000]
  949.    java.lang.Thread.State: WAITING (parking)
  950.     at sun.misc.Unsafe.park(Native Method)
  951.     - parking to wait for  <0x000000014ffa8498> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  952.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  953.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  954.     at java.util.concurrent.LinkedBlockingDeque.takeFirst(LinkedBlockingDeque.java:492)
  955.     at java.util.concurrent.LinkedBlockingDeque.take(LinkedBlockingDeque.java:680)
  956.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  957.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  958.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  959.     at java.lang.Thread.run(Thread.java:748)
  960.  
  961. "s3n-worker-9" #288 daemon prio=5 os_prio=0 tid=0x00007f9e18535800 nid=0x27d7 waiting on condition [0x00007f9ddd1b5000]
  962.    java.lang.Thread.State: WAITING (parking)
  963.     at sun.misc.Unsafe.park(Native Method)
  964.     - parking to wait for  <0x000000014ffa8498> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  965.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  966.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  967.     at java.util.concurrent.LinkedBlockingDeque.takeFirst(LinkedBlockingDeque.java:492)
  968.     at java.util.concurrent.LinkedBlockingDeque.take(LinkedBlockingDeque.java:680)
  969.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  970.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  971.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  972.     at java.lang.Thread.run(Thread.java:748)
  973.  
  974. "s3n-worker-8" #286 daemon prio=5 os_prio=0 tid=0x00007f9e18534000 nid=0x27d3 waiting on condition [0x00007f9ddd2b6000]
  975.    java.lang.Thread.State: WAITING (parking)
  976.     at sun.misc.Unsafe.park(Native Method)
  977.     - parking to wait for  <0x000000014ffa8498> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  978.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  979.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  980.     at java.util.concurrent.LinkedBlockingDeque.takeFirst(LinkedBlockingDeque.java:492)
  981.     at java.util.concurrent.LinkedBlockingDeque.take(LinkedBlockingDeque.java:680)
  982.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  983.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  984.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  985.     at java.lang.Thread.run(Thread.java:748)
  986.  
  987. "s3n-worker-7" #285 daemon prio=5 os_prio=0 tid=0x00007f9e18539000 nid=0x27d2 waiting on condition [0x00007f9ddd3b7000]
  988.    java.lang.Thread.State: WAITING (parking)
  989.     at sun.misc.Unsafe.park(Native Method)
  990.     - parking to wait for  <0x000000014ffa8498> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  991.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  992.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  993.     at java.util.concurrent.LinkedBlockingDeque.takeFirst(LinkedBlockingDeque.java:492)
  994.     at java.util.concurrent.LinkedBlockingDeque.take(LinkedBlockingDeque.java:680)
  995.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  996.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  997.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  998.     at java.lang.Thread.run(Thread.java:748)
  999.  
  1000. "sync.4" #282 prio=5 os_prio=0 tid=0x00007f9e20037800 nid=0x27cf waiting on condition [0x00007f9ddd5b9000]
  1001.    java.lang.Thread.State: WAITING (parking)
  1002.     at sun.misc.Unsafe.park(Native Method)
  1003.     - parking to wait for  <0x000000015354b140> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1004.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1005.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1006.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1007.     at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:1263)
  1008.     at java.lang.Thread.run(Thread.java:748)
  1009.  
  1010. "sync.3" #281 prio=5 os_prio=0 tid=0x00007f9e20036000 nid=0x27ce waiting on condition [0x00007f9ddd6ba000]
  1011.    java.lang.Thread.State: WAITING (parking)
  1012.     at sun.misc.Unsafe.park(Native Method)
  1013.     - parking to wait for  <0x0000000152c2e9a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1014.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1015.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1016.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1017.     at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:1263)
  1018.     at java.lang.Thread.run(Thread.java:748)
  1019.  
  1020. "sync.2" #280 prio=5 os_prio=0 tid=0x00007f9e20034000 nid=0x27cd waiting on condition [0x00007f9ddd7bb000]
  1021.    java.lang.Thread.State: WAITING (parking)
  1022.     at sun.misc.Unsafe.park(Native Method)
  1023.     - parking to wait for  <0x0000000152973768> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1024.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1025.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1026.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1027.     at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:1263)
  1028.     at java.lang.Thread.run(Thread.java:748)
  1029.  
  1030. "sync.1" #279 prio=5 os_prio=0 tid=0x00007f9e20033800 nid=0x27cc waiting on condition [0x00007f9ddd8bc000]
  1031.    java.lang.Thread.State: WAITING (parking)
  1032.     at sun.misc.Unsafe.park(Native Method)
  1033.     - parking to wait for  <0x000000015354b0f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1034.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1035.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1036.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1037.     at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:1263)
  1038.     at java.lang.Thread.run(Thread.java:748)
  1039.  
  1040. "sync.0" #278 prio=5 os_prio=0 tid=0x00007f9e20032800 nid=0x27cb waiting on condition [0x00007f9ddd9bd000]
  1041.    java.lang.Thread.State: WAITING (parking)
  1042.     at sun.misc.Unsafe.park(Native Method)
  1043.     - parking to wait for  <0x000000015354b0b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1044.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1045.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1046.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1047.     at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:1263)
  1048.     at java.lang.Thread.run(Thread.java:748)
  1049.  
  1050. "RS_OPEN_REGION-ip-172-20-113-118:16020-1.append-pool7-t1" #283 prio=5 os_prio=0 tid=0x00007f9e3763d000 nid=0x27ca waiting on condition [0x00007f9dddabe000]
  1051.    java.lang.Thread.State: WAITING (parking)
  1052.     at sun.misc.Unsafe.park(Native Method)
  1053.     - parking to wait for  <0x000000015354b068> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1054.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1055.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1056.     at com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:45)
  1057.     at com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:55)
  1058.     at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:123)
  1059.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  1060.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  1061.     at java.lang.Thread.run(Thread.java:748)
  1062.  
  1063. "RS_OPEN_REGION-ip-172-20-113-118:16020-1" #275 prio=5 os_prio=0 tid=0x00007f9e18005000 nid=0x27b2 waiting on condition [0x00007f9ddddc1000]
  1064.    java.lang.Thread.State: WAITING (parking)
  1065.     at sun.misc.Unsafe.park(Native Method)
  1066.     - parking to wait for  <0x0000000152c918e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1067.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1068.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1069.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1070.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  1071.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  1072.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  1073.     at java.lang.Thread.run(Thread.java:748)
  1074.  
  1075. "s3n-worker-6" #274 daemon prio=5 os_prio=0 tid=0x00007f9e1836c800 nid=0x27b1 waiting on condition [0x00007f9dddec2000]
  1076.    java.lang.Thread.State: WAITING (parking)
  1077.     at sun.misc.Unsafe.park(Native Method)
  1078.     - parking to wait for  <0x000000014ffa8498> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1079.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1080.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1081.     at java.util.concurrent.LinkedBlockingDeque.takeFirst(LinkedBlockingDeque.java:492)
  1082.     at java.util.concurrent.LinkedBlockingDeque.take(LinkedBlockingDeque.java:680)
  1083.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  1084.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  1085.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  1086.     at java.lang.Thread.run(Thread.java:748)
  1087.  
  1088. "Attach Listener" #273 daemon prio=9 os_prio=0 tid=0x00007f9e3731f000 nid=0x26ea waiting on condition [0x0000000000000000]
  1089.    java.lang.Thread.State: RUNNABLE
  1090.  
  1091. "s3n-worker-5" #266 daemon prio=5 os_prio=0 tid=0x00007f9e20698000 nid=0x25ae waiting on condition [0x00007f9dde6c6000]
  1092.    java.lang.Thread.State: WAITING (parking)
  1093.     at sun.misc.Unsafe.park(Native Method)
  1094.     - parking to wait for  <0x000000014ffa8498> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1095.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1096.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1097.     at java.util.concurrent.LinkedBlockingDeque.takeFirst(LinkedBlockingDeque.java:492)
  1098.     at java.util.concurrent.LinkedBlockingDeque.take(LinkedBlockingDeque.java:680)
  1099.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  1100.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  1101.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  1102.     at java.lang.Thread.run(Thread.java:748)
  1103.  
  1104. "s3n-worker-4" #265 daemon prio=5 os_prio=0 tid=0x00007f9e2049e800 nid=0x25ad waiting on condition [0x00007f9dde7c7000]
  1105.    java.lang.Thread.State: WAITING (parking)
  1106.     at sun.misc.Unsafe.park(Native Method)
  1107.     - parking to wait for  <0x000000014ffa8498> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1108.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1109.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1110.     at java.util.concurrent.LinkedBlockingDeque.takeFirst(LinkedBlockingDeque.java:492)
  1111.     at java.util.concurrent.LinkedBlockingDeque.take(LinkedBlockingDeque.java:680)
  1112.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  1113.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  1114.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  1115.     at java.lang.Thread.run(Thread.java:748)
  1116.  
  1117. "s3n-worker-3" #264 daemon prio=5 os_prio=0 tid=0x00007f9e20933000 nid=0x25ac waiting on condition [0x00007f9ddeaca000]
  1118.    java.lang.Thread.State: WAITING (parking)
  1119.     at sun.misc.Unsafe.park(Native Method)
  1120.     - parking to wait for  <0x000000014ffa8498> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1121.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1122.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1123.     at java.util.concurrent.LinkedBlockingDeque.takeFirst(LinkedBlockingDeque.java:492)
  1124.     at java.util.concurrent.LinkedBlockingDeque.take(LinkedBlockingDeque.java:680)
  1125.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  1126.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  1127.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  1128.     at java.lang.Thread.run(Thread.java:748)
  1129.  
  1130. "s3n-worker-2" #262 daemon prio=5 os_prio=0 tid=0x0000000001b17800 nid=0x25aa waiting on condition [0x00007f9dde8c8000]
  1131.    java.lang.Thread.State: WAITING (parking)
  1132.     at sun.misc.Unsafe.park(Native Method)
  1133.     - parking to wait for  <0x000000014ffa8498> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1134.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1135.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1136.     at java.util.concurrent.LinkedBlockingDeque.takeFirst(LinkedBlockingDeque.java:492)
  1137.     at java.util.concurrent.LinkedBlockingDeque.take(LinkedBlockingDeque.java:680)
  1138.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  1139.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  1140.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  1141.     at java.lang.Thread.run(Thread.java:748)
  1142.  
  1143. "s3n-worker-1" #261 daemon prio=5 os_prio=0 tid=0x0000000001ff6800 nid=0x25a9 waiting on condition [0x00007f9dde9c9000]
  1144.    java.lang.Thread.State: WAITING (parking)
  1145.     at sun.misc.Unsafe.park(Native Method)
  1146.     - parking to wait for  <0x000000014ffa8498> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1147.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1148.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1149.     at java.util.concurrent.LinkedBlockingDeque.takeFirst(LinkedBlockingDeque.java:492)
  1150.     at java.util.concurrent.LinkedBlockingDeque.take(LinkedBlockingDeque.java:680)
  1151.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  1152.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  1153.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  1154.     at java.lang.Thread.run(Thread.java:748)
  1155.  
  1156. "sync.4" #258 prio=5 os_prio=0 tid=0x00007f9e200ed000 nid=0x25a7 waiting on condition [0x00007f9ddefd0000]
  1157.    java.lang.Thread.State: WAITING (parking)
  1158.     at sun.misc.Unsafe.park(Native Method)
  1159.     - parking to wait for  <0x0000000152973750> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1160.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1161.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1162.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1163.     at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:1263)
  1164.     at java.lang.Thread.run(Thread.java:748)
  1165.  
  1166. "sync.3" #257 prio=5 os_prio=0 tid=0x00007f9e20770000 nid=0x25a6 waiting on condition [0x00007f9ddf0d1000]
  1167.    java.lang.Thread.State: WAITING (parking)
  1168.     at sun.misc.Unsafe.park(Native Method)
  1169.     - parking to wait for  <0x0000000152bdd378> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1170.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1171.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1172.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1173.     at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:1263)
  1174.     at java.lang.Thread.run(Thread.java:748)
  1175.  
  1176. "sync.2" #256 prio=5 os_prio=0 tid=0x00007f9e2076e000 nid=0x25a5 waiting on condition [0x00007f9ddf1d2000]
  1177.    java.lang.Thread.State: WAITING (parking)
  1178.     at sun.misc.Unsafe.park(Native Method)
  1179.     - parking to wait for  <0x0000000152c2e930> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1180.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1181.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1182.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1183.     at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:1263)
  1184.     at java.lang.Thread.run(Thread.java:748)
  1185.  
  1186. "sync.1" #255 prio=5 os_prio=0 tid=0x00007f9e2076c800 nid=0x25a4 waiting on condition [0x00007f9ddf2d3000]
  1187.    java.lang.Thread.State: WAITING (parking)
  1188.     at sun.misc.Unsafe.park(Native Method)
  1189.     - parking to wait for  <0x00000001529736d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1190.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1191.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1192.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1193.     at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:1263)
  1194.     at java.lang.Thread.run(Thread.java:748)
  1195.  
  1196. "sync.0" #254 prio=5 os_prio=0 tid=0x00007f9e20127000 nid=0x25a3 waiting on condition [0x00007f9ddf3d4000]
  1197.    java.lang.Thread.State: WAITING (parking)
  1198.     at sun.misc.Unsafe.park(Native Method)
  1199.     - parking to wait for  <0x0000000152bdd360> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1200.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1201.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1202.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1203.     at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:1263)
  1204.     at java.lang.Thread.run(Thread.java:748)
  1205.  
  1206. "RS_OPEN_REGION-ip-172-20-113-118:16020-0.append-pool5-t1" #259 prio=5 os_prio=0 tid=0x00007f9e20125000 nid=0x25a2 waiting on condition [0x00007f9ddf4d5000]
  1207.    java.lang.Thread.State: WAITING (parking)
  1208.     at sun.misc.Unsafe.park(Native Method)
  1209.     - parking to wait for  <0x0000000152c2e918> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1210.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1211.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1212.     at com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:45)
  1213.     at com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:55)
  1214.     at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:123)
  1215.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  1216.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  1217.     at java.lang.Thread.run(Thread.java:748)
  1218.  
  1219. "RS_OPEN_REGION-ip-172-20-113-118:16020-0" #250 prio=5 os_prio=0 tid=0x00007f9e184e2000 nid=0x259c waiting on condition [0x00007f9def184000]
  1220.    java.lang.Thread.State: WAITING (parking)
  1221.     at sun.misc.Unsafe.park(Native Method)
  1222.     - parking to wait for  <0x0000000152c918e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1223.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1224.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1225.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1226.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  1227.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  1228.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  1229.     at java.lang.Thread.run(Thread.java:748)
  1230.  
  1231. "s3n-worker-0" #249 daemon prio=5 os_prio=0 tid=0x00007f9e1875e000 nid=0x259b waiting on condition [0x00007f9de14ef000]
  1232.    java.lang.Thread.State: WAITING (parking)
  1233.     at sun.misc.Unsafe.park(Native Method)
  1234.     - parking to wait for  <0x000000014ffa8498> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1235.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1236.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1237.     at java.util.concurrent.LinkedBlockingDeque.takeFirst(LinkedBlockingDeque.java:492)
  1238.     at java.util.concurrent.LinkedBlockingDeque.take(LinkedBlockingDeque.java:680)
  1239.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  1240.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  1241.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  1242.     at java.lang.Thread.run(Thread.java:748)
  1243.  
  1244. "sync.4" #239 prio=5 os_prio=0 tid=0x00007f9e34402800 nid=0x2401 waiting on condition [0x00007f9ddfad9000]
  1245.    java.lang.Thread.State: WAITING (parking)
  1246.     at sun.misc.Unsafe.park(Native Method)
  1247.     - parking to wait for  <0x0000000151124f60> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1248.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1249.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1250.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1251.     at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:1263)
  1252.     at java.lang.Thread.run(Thread.java:748)
  1253.  
  1254. "sync.3" #238 prio=5 os_prio=0 tid=0x00007f9e34595000 nid=0x2400 waiting on condition [0x00007f9ddfbda000]
  1255.    java.lang.Thread.State: WAITING (parking)
  1256.     at sun.misc.Unsafe.park(Native Method)
  1257.     - parking to wait for  <0x0000000151236f80> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1258.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1259.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1260.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1261.     at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:1263)
  1262.     at java.lang.Thread.run(Thread.java:748)
  1263.  
  1264. "sync.2" #237 prio=5 os_prio=0 tid=0x00007f9e344da000 nid=0x23ff waiting on condition [0x00007f9ddfcdb000]
  1265.    java.lang.Thread.State: WAITING (parking)
  1266.     at sun.misc.Unsafe.park(Native Method)
  1267.     - parking to wait for  <0x0000000150fccdf0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1268.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1269.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1270.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1271.     at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:1263)
  1272.     at java.lang.Thread.run(Thread.java:748)
  1273.  
  1274. "sync.1" #236 prio=5 os_prio=0 tid=0x00007f9e34785800 nid=0x23fe waiting on condition [0x00007f9ddfddc000]
  1275.    java.lang.Thread.State: WAITING (parking)
  1276.     at sun.misc.Unsafe.park(Native Method)
  1277.     - parking to wait for  <0x0000000150fb6680> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1278.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1279.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1280.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1281.     at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:1263)
  1282.     at java.lang.Thread.run(Thread.java:748)
  1283.  
  1284. "sync.0" #235 prio=5 os_prio=0 tid=0x00007f9e37591000 nid=0x23fd waiting on condition [0x00007f9ddfedd000]
  1285.    java.lang.Thread.State: WAITING (parking)
  1286.     at sun.misc.Unsafe.park(Native Method)
  1287.     - parking to wait for  <0x0000000151339780> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1288.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1289.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1290.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1291.     at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:1263)
  1292.     at java.lang.Thread.run(Thread.java:748)
  1293.  
  1294. "regionserver/ip-172-20-113-118.us-west-2.compute.internal/172.20.113.118:16020.append-pool4-t1" #240 prio=5 os_prio=0 tid=0x00007f9e20332000 nid=0x23fc waiting on condition [0x00007f9ddffde000]
  1295.    java.lang.Thread.State: WAITING (parking)
  1296.     at sun.misc.Unsafe.park(Native Method)
  1297.     - parking to wait for  <0x00000001513397c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1298.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1299.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1300.     at com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:45)
  1301.     at com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:55)
  1302.     at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:123)
  1303.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  1304.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  1305.     at java.lang.Thread.run(Thread.java:748)
  1306.  
  1307. "LeaseRenewer:hbase@ip-172-20-113-124.us-west-2.compute.internal:8020" #233 daemon prio=5 os_prio=0 tid=0x00007f9e20669000 nid=0x23f8 waiting on condition [0x00007f9de03e0000]
  1308.    java.lang.Thread.State: TIMED_WAITING (sleeping)
  1309.     at java.lang.Thread.sleep(Native Method)
  1310.     at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:411)
  1311.     at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$600(LeaseRenewer.java:76)
  1312.     at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:307)
  1313.     at java.lang.Thread.run(Thread.java:748)
  1314.  
  1315. "member: 'ip-172-20-113-118.us-west-2.compute.internal,16020,1568477753544' subprocedure-pool2-thread-1" #231 daemon prio=5 os_prio=0 tid=0x00007f9e2089b800 nid=0x23f0 waiting on condition [0x00007f9de05e2000]
  1316.    java.lang.Thread.State: WAITING (parking)
  1317.     at sun.misc.Unsafe.park(Native Method)
  1318.     - parking to wait for  <0x00000001534d5d98> (a java.util.concurrent.SynchronousQueue$TransferStack)
  1319.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1320.     at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458)
  1321.     at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
  1322.     at java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924)
  1323.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  1324.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  1325.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  1326.     at java.lang.Thread.run(Thread.java:748)
  1327.  
  1328. "SplitLogWorker-ip-172-20-113-118:16020" #229 prio=5 os_prio=0 tid=0x00007f9e208e4800 nid=0x23ee in Object.wait() [0x00007f9de07e4000]
  1329.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  1330.     at java.lang.Object.wait(Native Method)
  1331.     at org.apache.hadoop.hbase.coordination.ZkSplitLogWorkerCoordination.taskLoop(ZkSplitLogWorkerCoordination.java:445)
  1332.     - locked <0x0000000150f79448> (a java.lang.Object)
  1333.     at org.apache.hadoop.hbase.regionserver.SplitLogWorker.run(SplitLogWorker.java:148)
  1334.     at java.lang.Thread.run(Thread.java:748)
  1335.  
  1336. "ip-172-20-113-118:16020Replication Statistics #0" #228 daemon prio=5 os_prio=0 tid=0x00007f9e208ed000 nid=0x23ed waiting on condition [0x00007f9de08e5000]
  1337.    java.lang.Thread.State: TIMED_WAITING (parking)
  1338.     at sun.misc.Unsafe.park(Native Method)
  1339.     - parking to wait for  <0x0000000151124fa8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1340.     at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
  1341.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
  1342.     at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
  1343.     at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
  1344.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  1345.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  1346.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  1347.     at java.lang.Thread.run(Thread.java:748)
  1348.  
  1349. "ReplicationExecutor-0" #223 daemon prio=5 os_prio=0 tid=0x00007f9e2088a800 nid=0x23ec waiting on condition [0x00007f9de0be6000]
  1350.    java.lang.Thread.State: WAITING (parking)
  1351.     at sun.misc.Unsafe.park(Native Method)
  1352.     - parking to wait for  <0x0000000151236ec0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1353.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1354.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1355.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1356.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  1357.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  1358.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  1359.     at java.lang.Thread.run(Thread.java:748)
  1360.  
  1361. "regionserver/ip-172-20-113-118.us-west-2.compute.internal/172.20.113.118:16020.leaseChecker" #197 daemon prio=5 os_prio=0 tid=0x00007f9e20888000 nid=0x23eb waiting on condition [0x00007f9de0ce7000]
  1362.    java.lang.Thread.State: TIMED_WAITING (sleeping)
  1363.     at java.lang.Thread.sleep(Native Method)
  1364.     at org.apache.hadoop.hbase.regionserver.Leases.run(Leases.java:95)
  1365.     at java.lang.Thread.run(Thread.java:748)
  1366.  
  1367. "MemStoreFlusher.3" #220 prio=5 os_prio=0 tid=0x00007f9e20886800 nid=0x23ea waiting on condition [0x00007f9de0de8000]
  1368.    java.lang.Thread.State: TIMED_WAITING (parking)
  1369.     at sun.misc.Unsafe.park(Native Method)
  1370.     - parking to wait for  <0x0000000150f21370> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1371.     at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
  1372.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
  1373.     at java.util.concurrent.DelayQueue.poll(DelayQueue.java:268)
  1374.     at java.util.concurrent.DelayQueue.poll(DelayQueue.java:70)
  1375.     at org.apache.hadoop.hbase.regionserver.MemStoreFlusher$FlushHandler.run(MemStoreFlusher.java:244)
  1376.     at java.lang.Thread.run(Thread.java:748)
  1377.  
  1378. "MemStoreFlusher.2" #218 prio=5 os_prio=0 tid=0x00007f9e2087c800 nid=0x23e9 waiting on condition [0x00007f9de0ee9000]
  1379.    java.lang.Thread.State: TIMED_WAITING (parking)
  1380.     at sun.misc.Unsafe.park(Native Method)
  1381.     - parking to wait for  <0x0000000150f21370> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1382.     at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
  1383.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
  1384.     at java.util.concurrent.DelayQueue.poll(DelayQueue.java:268)
  1385.     at java.util.concurrent.DelayQueue.poll(DelayQueue.java:70)
  1386.     at org.apache.hadoop.hbase.regionserver.MemStoreFlusher$FlushHandler.run(MemStoreFlusher.java:244)
  1387.     at java.lang.Thread.run(Thread.java:748)
  1388.  
  1389. "MemStoreFlusher.1" #216 prio=5 os_prio=0 tid=0x00007f9e2087a800 nid=0x23e8 waiting on condition [0x00007f9de0fea000]
  1390.    java.lang.Thread.State: TIMED_WAITING (parking)
  1391.     at sun.misc.Unsafe.park(Native Method)
  1392.     - parking to wait for  <0x0000000150f21370> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1393.     at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
  1394.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
  1395.     at java.util.concurrent.DelayQueue.poll(DelayQueue.java:268)
  1396.     at java.util.concurrent.DelayQueue.poll(DelayQueue.java:70)
  1397.     at org.apache.hadoop.hbase.regionserver.MemStoreFlusher$FlushHandler.run(MemStoreFlusher.java:244)
  1398.     at java.lang.Thread.run(Thread.java:748)
  1399.  
  1400. "MemStoreFlusher.0" #214 prio=5 os_prio=0 tid=0x00007f9e20879800 nid=0x23e7 waiting on condition [0x00007f9de10eb000]
  1401.    java.lang.Thread.State: TIMED_WAITING (parking)
  1402.     at sun.misc.Unsafe.park(Native Method)
  1403.     - parking to wait for  <0x0000000150f21370> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1404.     at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
  1405.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
  1406.     at java.util.concurrent.DelayQueue.poll(DelayQueue.java:273)
  1407.     at java.util.concurrent.DelayQueue.poll(DelayQueue.java:70)
  1408.     at org.apache.hadoop.hbase.regionserver.MemStoreFlusher$FlushHandler.run(MemStoreFlusher.java:244)
  1409.     at java.lang.Thread.run(Thread.java:748)
  1410.  
  1411. "regionserver/ip-172-20-113-118.us-west-2.compute.internal/172.20.113.118:16020.logRoller" #190 daemon prio=5 os_prio=0 tid=0x00007f9e20878800 nid=0x23e6 in Object.wait() [0x00007f9de11ec000]
  1412.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  1413.     at java.lang.Object.wait(Native Method)
  1414.     at org.apache.hadoop.hbase.regionserver.LogRoller.run(LogRoller.java:151)
  1415.     - locked <0x00000001513396f0> (a java.util.concurrent.atomic.AtomicBoolean)
  1416.     at java.lang.Thread.run(Thread.java:748)
  1417.  
  1418. "JvmPauseMonitor" #213 daemon prio=5 os_prio=0 tid=0x00007f9e20873800 nid=0x23e5 waiting on condition [0x00007f9de12ed000]
  1419.    java.lang.Thread.State: TIMED_WAITING (sleeping)
  1420.     at java.lang.Thread.sleep(Native Method)
  1421.     at org.apache.hadoop.hbase.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:153)
  1422.     at java.lang.Thread.run(Thread.java:748)
  1423.  
  1424. "IPC Parameter Sending Thread #0" #212 daemon prio=5 os_prio=0 tid=0x00007f9e20712800 nid=0x23e4 waiting on condition [0x00007f9de13ee000]
  1425.    java.lang.Thread.State: TIMED_WAITING (parking)
  1426.     at sun.misc.Unsafe.park(Native Method)
  1427.     - parking to wait for  <0x0000000150f793d0> (a java.util.concurrent.SynchronousQueue$TransferStack)
  1428.     at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
  1429.     at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
  1430.     at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
  1431.     at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
  1432.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
  1433.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  1434.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  1435.     at java.lang.Thread.run(Thread.java:748)
  1436.  
  1437. "BucketCacheStatsExecutor" #210 daemon prio=5 os_prio=0 tid=0x00007f9e2075f800 nid=0x23e2 waiting on condition [0x00007f9de15f0000]
  1438.    java.lang.Thread.State: TIMED_WAITING (parking)
  1439.     at sun.misc.Unsafe.park(Native Method)
  1440.     - parking to wait for  <0x0000000150f213a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1441.     at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
  1442.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
  1443.     at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
  1444.     at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
  1445.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  1446.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  1447.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  1448.     at java.lang.Thread.run(Thread.java:748)
  1449.  
  1450. "regionserver/ip-172-20-113-118.us-west-2.compute.internal/172.20.113.118:16020-BucketCacheWriter-5" #208 daemon prio=5 os_prio=0 tid=0x00007f9e2075d800 nid=0x23e1 waiting on condition [0x00007f9de16f1000]
  1451.    java.lang.Thread.State: WAITING (parking)
  1452.     at sun.misc.Unsafe.park(Native Method)
  1453.     - parking to wait for  <0x0000000151236f08> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1454.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1455.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1456.     at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)
  1457.     at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.getRAMQueueEntries(BucketCache.java:1017)
  1458.     at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache$WriterThread.run(BucketCache.java:853)
  1459.     at java.lang.Thread.run(Thread.java:748)
  1460.  
  1461. "regionserver/ip-172-20-113-118.us-west-2.compute.internal/172.20.113.118:16020-BucketCacheWriter-4" #207 daemon prio=5 os_prio=0 tid=0x00007f9e2075b800 nid=0x23e0 waiting on condition [0x00007f9de17f2000]
  1462.    java.lang.Thread.State: WAITING (parking)
  1463.     at sun.misc.Unsafe.park(Native Method)
  1464.     - parking to wait for  <0x0000000149f0b5c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1465.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1466.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1467.     at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)
  1468.     at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.getRAMQueueEntries(BucketCache.java:1017)
  1469.     at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache$WriterThread.run(BucketCache.java:853)
  1470.     at java.lang.Thread.run(Thread.java:748)
  1471.  
  1472. "regionserver/ip-172-20-113-118.us-west-2.compute.internal/172.20.113.118:16020-BucketCacheWriter-3" #206 daemon prio=5 os_prio=0 tid=0x00007f9e20759800 nid=0x23df waiting on condition [0x00007f9de18f3000]
  1473.    java.lang.Thread.State: WAITING (parking)
  1474.     at sun.misc.Unsafe.park(Native Method)
  1475.     - parking to wait for  <0x0000000151339708> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1476.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1477.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1478.     at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)
  1479.     at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.getRAMQueueEntries(BucketCache.java:1017)
  1480.     at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache$WriterThread.run(BucketCache.java:853)
  1481.     at java.lang.Thread.run(Thread.java:748)
  1482.  
  1483. "regionserver/ip-172-20-113-118.us-west-2.compute.internal/172.20.113.118:16020-BucketCacheWriter-2" #205 daemon prio=5 os_prio=0 tid=0x00007f9e20758000 nid=0x23de waiting on condition [0x00007f9de19f4000]
  1484.    java.lang.Thread.State: WAITING (parking)
  1485.     at sun.misc.Unsafe.park(Native Method)
  1486.     - parking to wait for  <0x0000000151124ed0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1487.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1488.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1489.     at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)
  1490.     at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.getRAMQueueEntries(BucketCache.java:1017)
  1491.     at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache$WriterThread.run(BucketCache.java:853)
  1492.     at java.lang.Thread.run(Thread.java:748)
  1493.  
  1494. "regionserver/ip-172-20-113-118.us-west-2.compute.internal/172.20.113.118:16020-BucketCacheWriter-1" #204 daemon prio=5 os_prio=0 tid=0x00007f9e20756000 nid=0x23dd waiting on condition [0x00007f9de1af5000]
  1495.    java.lang.Thread.State: WAITING (parking)
  1496.     at sun.misc.Unsafe.park(Native Method)
  1497.     - parking to wait for  <0x0000000151237a30> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1498.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1499.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1500.     at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)
  1501.     at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.getRAMQueueEntries(BucketCache.java:1017)
  1502.     at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache$WriterThread.run(BucketCache.java:853)
  1503.     at java.lang.Thread.run(Thread.java:748)
  1504.  
  1505. "regionserver/ip-172-20-113-118.us-west-2.compute.internal/172.20.113.118:16020-BucketCacheWriter-0" #203 daemon prio=5 os_prio=0 tid=0x00007f9e20751000 nid=0x23dc waiting on condition [0x00007f9de1bf6000]
  1506.    java.lang.Thread.State: WAITING (parking)
  1507.     at sun.misc.Unsafe.park(Native Method)
  1508.     - parking to wait for  <0x0000000151124e58> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1509.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1510.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1511.     at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403)
  1512.     at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.getRAMQueueEntries(BucketCache.java:1017)
  1513.     at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache$WriterThread.run(BucketCache.java:853)
  1514.     at java.lang.Thread.run(Thread.java:748)
  1515.  
  1516. "LruBlockCacheStatsExecutor" #202 daemon prio=5 os_prio=0 tid=0x00007f9e20746800 nid=0x23da waiting on condition [0x00007f9de1cf7000]
  1517.    java.lang.Thread.State: TIMED_WAITING (parking)
  1518.     at sun.misc.Unsafe.park(Native Method)
  1519.     - parking to wait for  <0x000000015133a0b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1520.     at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
  1521.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
  1522.     at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
  1523.     at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
  1524.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  1525.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  1526.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  1527.     at java.lang.Thread.run(Thread.java:748)
  1528.  
  1529. "regionserver/ip-172-20-113-118.us-west-2.compute.internal/172.20.113.118:16020.LruBlockCache.EvictionThread" #200 daemon prio=5 os_prio=0 tid=0x00007f9e20749000 nid=0x23d9 in Object.wait() [0x00007f9de1df8000]
  1530.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  1531.     at java.lang.Object.wait(Native Method)
  1532.     at org.apache.hadoop.hbase.io.hfile.LruBlockCache$EvictionThread.run(LruBlockCache.java:883)
  1533.     - locked <0x0000000150f793e8> (a org.apache.hadoop.hbase.io.hfile.LruBlockCache$EvictionThread)
  1534.     at java.lang.Thread.run(Thread.java:748)
  1535.  
  1536. "RpcClient-timer-pool1-t1" #194 daemon prio=5 os_prio=0 tid=0x00007f9e206e8000 nid=0x23d7 waiting on condition [0x00007f9de1ffa000]
  1537.    java.lang.Thread.State: TIMED_WAITING (sleeping)
  1538.     at java.lang.Thread.sleep(Native Method)
  1539.     at io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:522)
  1540.     at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:421)
  1541.     at java.lang.Thread.run(Thread.java:748)
  1542.  
  1543. "regionserver/ip-172-20-113-118.us-west-2.compute.internal/172.20.113.118:16020-longCompactions-1568477759662" #196 prio=5 os_prio=0 tid=0x00007f9e204a5800 nid=0x23d6 waiting on condition [0x00007f9de22fb000]
  1544.    java.lang.Thread.State: WAITING (parking)
  1545.     at sun.misc.Unsafe.park(Native Method)
  1546.     - parking to wait for  <0x00000001512379b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1547.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1548.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1549.     at org.apache.hadoop.hbase.util.StealJobQueue.take(StealJobQueue.java:112)
  1550.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  1551.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  1552.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  1553.     at java.lang.Thread.run(Thread.java:748)
  1554.  
  1555. "Idle-Rpc-Conn-Sweeper-pool2-t1" #195 daemon prio=5 os_prio=0 tid=0x00007f9e203cd000 nid=0x23d5 waiting on condition [0x00007f9de23fc000]
  1556.    java.lang.Thread.State: TIMED_WAITING (parking)
  1557.     at sun.misc.Unsafe.park(Native Method)
  1558.     - parking to wait for  <0x0000000151124de0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1559.     at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
  1560.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
  1561.     at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
  1562.     at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
  1563.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  1564.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  1565.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  1566.     at java.lang.Thread.run(Thread.java:748)
  1567.  
  1568. "regionserver/ip-172-20-113-118.us-west-2.compute.internal/172.20.113.118:16020-EventThread" #193 daemon prio=5 os_prio=0 tid=0x00007f9e2067a000 nid=0x23d4 waiting on condition [0x00007f9de24fd000]
  1569.    java.lang.Thread.State: WAITING (parking)
  1570.     at sun.misc.Unsafe.park(Native Method)
  1571.     - parking to wait for  <0x0000000150f79400> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1572.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1573.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1574.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1575.     at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:504)
  1576.  
  1577. "regionserver/ip-172-20-113-118.us-west-2.compute.internal/172.20.113.118:16020-SendThread(ip-172-20-113-124.us-west-2.compute.internal:2181)" #192 daemon prio=5 os_prio=0 tid=0x00007f9e2067b800 nid=0x23d3 runnable [0x00007f9de25fe000]
  1578.    java.lang.Thread.State: RUNNABLE
  1579.     at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  1580.     at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
  1581.     at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
  1582.     at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
  1583.     - locked <0x000000015133a038> (a sun.nio.ch.Util$3)
  1584.     - locked <0x000000015133a020> (a java.util.Collections$UnmodifiableSet)
  1585.     - locked <0x000000014a0f3fd8> (a sun.nio.ch.EPollSelectorImpl)
  1586.     at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
  1587.     at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)
  1588.     at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1145)
  1589.  
  1590. "regionserver/ip-172-20-113-118.us-west-2.compute.internal/172.20.113.118:16020" #14 prio=5 os_prio=0 tid=0x00007f9e3771e000 nid=0x23d2 in Object.wait() [0x00007f9de28ff000]
  1591.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  1592.     at java.lang.Object.wait(Native Method)
  1593.     at org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:92)
  1594.     - locked <0x0000000149f47bd0> (a java.lang.Object)
  1595.     at org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:56)
  1596.     at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:1028)
  1597.     at java.lang.Thread.run(Thread.java:748)
  1598.  
  1599. "ip-172-20-113-118.us-west-2.compute.internal,16020,1568477753544_ChoreService_1" #191 daemon prio=5 os_prio=0 tid=0x00007f9e3771c800 nid=0x23d1 waiting on condition [0x00007f9de2a00000]
  1600.    java.lang.Thread.State: WAITING (parking)
  1601.     at sun.misc.Unsafe.park(Native Method)
  1602.     - parking to wait for  <0x0000000150f213d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1603.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1604.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1605.     at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)
  1606.     at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
  1607.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  1608.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  1609.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  1610.     at java.lang.Thread.run(Thread.java:748)
  1611.  
  1612. "Timer-0" #189 daemon prio=5 os_prio=0 tid=0x00007f9e376c6800 nid=0x23d0 in Object.wait() [0x00007f9de2b01000]
  1613.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  1614.     at java.lang.Object.wait(Native Method)
  1615.     at java.util.TimerThread.mainLoop(Timer.java:552)
  1616.     - locked <0x0000000151124e10> (a java.util.TaskQueue)
  1617.     at java.util.TimerThread.run(Timer.java:505)
  1618.  
  1619. "Monitor thread for TaskMonitor" #188 daemon prio=5 os_prio=0 tid=0x0000000002344800 nid=0x23cf waiting on condition [0x00007f9de2c02000]
  1620.    java.lang.Thread.State: TIMED_WAITING (sleeping)
  1621.     at java.lang.Thread.sleep(Native Method)
  1622.     at org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:273)
  1623.     at java.lang.Thread.run(Thread.java:748)
  1624.  
  1625. "1269938539@qtp-1872627924-1 - Acceptor0 SelectChannelConnector@0.0.0.0:16030" #187 daemon prio=5 os_prio=0 tid=0x00007f9e3758a800 nid=0x23ce runnable [0x00007f9de2d03000]
  1626.    java.lang.Thread.State: RUNNABLE
  1627.     at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  1628.     at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
  1629.     at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
  1630.     at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
  1631.     - locked <0x0000000150f793a0> (a sun.nio.ch.Util$3)
  1632.     - locked <0x0000000150f79388> (a java.util.Collections$UnmodifiableSet)
  1633.     - locked <0x000000014a0eb4e0> (a sun.nio.ch.EPollSelectorImpl)
  1634.     at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
  1635.     at org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498)
  1636.     at org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192)
  1637.     at org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124)
  1638.     at org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708)
  1639.     at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582)
  1640.  
  1641. "1726054143@qtp-1872627924-0" #186 daemon prio=5 os_prio=0 tid=0x00007f9e3757e000 nid=0x23cd in Object.wait() [0x00007f9de2e04000]
  1642.    java.lang.Thread.State: TIMED_WAITING (on object monitor)
  1643.     at java.lang.Object.wait(Native Method)
  1644.     at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626)
  1645.     - locked <0x00000001513874f8> (a org.mortbay.thread.QueuedThreadPool$PoolThread)
  1646.  
  1647. "RpcServer.Metadata.handler=29,queue=0,port=16020" #184 daemon prio=5 os_prio=0 tid=0x00007f9e37500000 nid=0x23cc waiting on condition [0x00007f9de3105000]
  1648.    java.lang.Thread.State: WAITING (parking)
  1649.     at sun.misc.Unsafe.park(Native Method)
  1650.     - parking to wait for  <0x0000000149b65cc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1651.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1652.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1653.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1654.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  1655.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  1656.  
  1657. "RpcServer.Metadata.handler=28,queue=0,port=16020" #183 daemon prio=5 os_prio=0 tid=0x00007f9e374fd800 nid=0x23cb waiting on condition [0x00007f9de3206000]
  1658.    java.lang.Thread.State: WAITING (parking)
  1659.     at sun.misc.Unsafe.park(Native Method)
  1660.     - parking to wait for  <0x0000000149b65cc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1661.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1662.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1663.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1664.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  1665.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  1666.  
  1667. "RpcServer.Metadata.handler=27,queue=0,port=16020" #182 daemon prio=5 os_prio=0 tid=0x00007f9e374fb800 nid=0x23ca waiting on condition [0x00007f9de3307000]
  1668.    java.lang.Thread.State: WAITING (parking)
  1669.     at sun.misc.Unsafe.park(Native Method)
  1670.     - parking to wait for  <0x0000000149b65cc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1671.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1672.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1673.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1674.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  1675.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  1676.  
  1677. "RpcServer.Metadata.handler=26,queue=0,port=16020" #181 daemon prio=5 os_prio=0 tid=0x00007f9e374f9800 nid=0x23c9 waiting on condition [0x00007f9de3408000]
  1678.    java.lang.Thread.State: WAITING (parking)
  1679.     at sun.misc.Unsafe.park(Native Method)
  1680.     - parking to wait for  <0x0000000149b65cc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1681.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1682.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1683.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1684.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  1685.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  1686.  
  1687. "RpcServer.Metadata.handler=25,queue=0,port=16020" #180 daemon prio=5 os_prio=0 tid=0x00007f9e374f7800 nid=0x23c8 waiting on condition [0x00007f9de3509000]
  1688.    java.lang.Thread.State: WAITING (parking)
  1689.     at sun.misc.Unsafe.park(Native Method)
  1690.     - parking to wait for  <0x0000000149b65cc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1691.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1692.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1693.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1694.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  1695.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  1696.  
  1697. "RpcServer.Metadata.handler=24,queue=0,port=16020" #179 daemon prio=5 os_prio=0 tid=0x00007f9e374f5800 nid=0x23c7 waiting on condition [0x00007f9de360a000]
  1698.    java.lang.Thread.State: WAITING (parking)
  1699.     at sun.misc.Unsafe.park(Native Method)
  1700.     - parking to wait for  <0x0000000149b65cc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1701.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1702.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1703.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1704.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  1705.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  1706.  
  1707. "RpcServer.Metadata.handler=23,queue=0,port=16020" #178 daemon prio=5 os_prio=0 tid=0x00007f9e374f3000 nid=0x23c6 waiting on condition [0x00007f9de370b000]
  1708.    java.lang.Thread.State: WAITING (parking)
  1709.     at sun.misc.Unsafe.park(Native Method)
  1710.     - parking to wait for  <0x0000000149b65cc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1711.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1712.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1713.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1714.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  1715.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  1716.  
  1717. "RpcServer.Metadata.handler=22,queue=0,port=16020" #177 daemon prio=5 os_prio=0 tid=0x00007f9e374f1000 nid=0x23c5 waiting on condition [0x00007f9de380c000]
  1718.    java.lang.Thread.State: WAITING (parking)
  1719.     at sun.misc.Unsafe.park(Native Method)
  1720.     - parking to wait for  <0x0000000149b65cc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1721.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1722.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1723.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1724.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  1725.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  1726.  
  1727. "RpcServer.Metadata.handler=21,queue=0,port=16020" #176 daemon prio=5 os_prio=0 tid=0x00007f9e374ef000 nid=0x23c4 waiting on condition [0x00007f9de390d000]
  1728.    java.lang.Thread.State: WAITING (parking)
  1729.     at sun.misc.Unsafe.park(Native Method)
  1730.     - parking to wait for  <0x0000000149b65cc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1731.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1732.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1733.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1734.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  1735.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  1736.  
  1737. "RpcServer.Metadata.handler=20,queue=0,port=16020" #175 daemon prio=5 os_prio=0 tid=0x00007f9e374ed000 nid=0x23c3 waiting on condition [0x00007f9de3a0e000]
  1738.    java.lang.Thread.State: WAITING (parking)
  1739.     at sun.misc.Unsafe.park(Native Method)
  1740.     - parking to wait for  <0x0000000149b65cc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1741.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1742.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1743.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1744.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  1745.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  1746.  
  1747. "RpcServer.Metadata.handler=19,queue=0,port=16020" #174 daemon prio=5 os_prio=0 tid=0x00007f9e374eb000 nid=0x23c2 waiting on condition [0x00007f9de3b0f000]
  1748.    java.lang.Thread.State: WAITING (parking)
  1749.     at sun.misc.Unsafe.park(Native Method)
  1750.     - parking to wait for  <0x0000000149b65cc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1751.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1752.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1753.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1754.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  1755.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  1756.  
  1757. "RpcServer.Metadata.handler=18,queue=0,port=16020" #173 daemon prio=5 os_prio=0 tid=0x00007f9e374e9000 nid=0x23c1 waiting on condition [0x00007f9de3c10000]
  1758.    java.lang.Thread.State: WAITING (parking)
  1759.     at sun.misc.Unsafe.park(Native Method)
  1760.     - parking to wait for  <0x0000000149b65cc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1761.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1762.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1763.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1764.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  1765.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  1766.  
  1767. "RpcServer.Metadata.handler=17,queue=0,port=16020" #172 daemon prio=5 os_prio=0 tid=0x00007f9e374e6800 nid=0x23c0 waiting on condition [0x00007f9de3d11000]
  1768.    java.lang.Thread.State: WAITING (parking)
  1769.     at sun.misc.Unsafe.park(Native Method)
  1770.     - parking to wait for  <0x0000000149b65cc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1771.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1772.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1773.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1774.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  1775.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  1776.  
  1777. "RpcServer.Metadata.handler=16,queue=0,port=16020" #171 daemon prio=5 os_prio=0 tid=0x00007f9e374e4800 nid=0x23bf waiting on condition [0x00007f9de3e12000]
  1778.    java.lang.Thread.State: WAITING (parking)
  1779.     at sun.misc.Unsafe.park(Native Method)
  1780.     - parking to wait for  <0x0000000149b65cc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1781.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1782.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1783.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1784.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  1785.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  1786.  
  1787. "RpcServer.Metadata.handler=15,queue=0,port=16020" #170 daemon prio=5 os_prio=0 tid=0x00007f9e374e2800 nid=0x23be waiting on condition [0x00007f9de3f13000]
  1788.    java.lang.Thread.State: WAITING (parking)
  1789.     at sun.misc.Unsafe.park(Native Method)
  1790.     - parking to wait for  <0x0000000149b65cc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1791.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1792.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1793.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1794.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  1795.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  1796.  
  1797. "RpcServer.Metadata.handler=14,queue=0,port=16020" #169 daemon prio=5 os_prio=0 tid=0x00007f9e374e0800 nid=0x23bd waiting on condition [0x00007f9de4014000]
  1798.    java.lang.Thread.State: WAITING (parking)
  1799.     at sun.misc.Unsafe.park(Native Method)
  1800.     - parking to wait for  <0x0000000149b65cc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1801.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1802.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1803.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1804.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  1805.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  1806.  
  1807. "RpcServer.Metadata.handler=13,queue=0,port=16020" #168 daemon prio=5 os_prio=0 tid=0x00007f9e374df000 nid=0x23bc waiting on condition [0x00007f9de4115000]
  1808.    java.lang.Thread.State: WAITING (parking)
  1809.     at sun.misc.Unsafe.park(Native Method)
  1810.     - parking to wait for  <0x0000000149b65cc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1811.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1812.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1813.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1814.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  1815.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  1816.  
  1817. "RpcServer.Metadata.handler=12,queue=0,port=16020" #167 daemon prio=5 os_prio=0 tid=0x00007f9e374dd000 nid=0x23bb waiting on condition [0x00007f9de4216000]
  1818.    java.lang.Thread.State: WAITING (parking)
  1819.     at sun.misc.Unsafe.park(Native Method)
  1820.     - parking to wait for  <0x0000000149b65cc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1821.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1822.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1823.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1824.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  1825.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  1826.  
  1827. "RpcServer.Metadata.handler=11,queue=0,port=16020" #166 daemon prio=5 os_prio=0 tid=0x00007f9e374db000 nid=0x23ba waiting on condition [0x00007f9de4317000]
  1828.    java.lang.Thread.State: WAITING (parking)
  1829.     at sun.misc.Unsafe.park(Native Method)
  1830.     - parking to wait for  <0x0000000149b65cc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1831.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1832.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1833.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1834.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  1835.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  1836.  
  1837. "RpcServer.Metadata.handler=10,queue=0,port=16020" #165 daemon prio=5 os_prio=0 tid=0x00007f9e374d9800 nid=0x23b9 waiting on condition [0x00007f9de4418000]
  1838.    java.lang.Thread.State: WAITING (parking)
  1839.     at sun.misc.Unsafe.park(Native Method)
  1840.     - parking to wait for  <0x0000000149b65cc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1841.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1842.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1843.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1844.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  1845.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  1846.  
  1847. "RpcServer.Metadata.handler=9,queue=0,port=16020" #164 daemon prio=5 os_prio=0 tid=0x00007f9e374d7000 nid=0x23b8 waiting on condition [0x00007f9de4519000]
  1848.    java.lang.Thread.State: WAITING (parking)
  1849.     at sun.misc.Unsafe.park(Native Method)
  1850.     - parking to wait for  <0x0000000149b65cc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1851.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1852.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1853.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1854.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  1855.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  1856.  
  1857. "RpcServer.Metadata.handler=8,queue=0,port=16020" #163 daemon prio=5 os_prio=0 tid=0x00007f9e374d5000 nid=0x23b7 waiting on condition [0x00007f9de461a000]
  1858.    java.lang.Thread.State: WAITING (parking)
  1859.     at sun.misc.Unsafe.park(Native Method)
  1860.     - parking to wait for  <0x0000000149b65cc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1861.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1862.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1863.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1864.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  1865.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  1866.  
  1867. "RpcServer.Metadata.handler=7,queue=0,port=16020" #162 daemon prio=5 os_prio=0 tid=0x00007f9e374d3800 nid=0x23b6 waiting on condition [0x00007f9de471b000]
  1868.    java.lang.Thread.State: WAITING (parking)
  1869.     at sun.misc.Unsafe.park(Native Method)
  1870.     - parking to wait for  <0x0000000149b65cc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1871.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1872.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1873.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1874.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  1875.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  1876.  
  1877. "RpcServer.Metadata.handler=6,queue=0,port=16020" #161 daemon prio=5 os_prio=0 tid=0x00007f9e374d1800 nid=0x23b5 waiting on condition [0x00007f9de481c000]
  1878.    java.lang.Thread.State: WAITING (parking)
  1879.     at sun.misc.Unsafe.park(Native Method)
  1880.     - parking to wait for  <0x0000000149b65cc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1881.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1882.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1883.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1884.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  1885.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  1886.  
  1887. "RpcServer.Metadata.handler=5,queue=0,port=16020" #160 daemon prio=5 os_prio=0 tid=0x00007f9e374cf000 nid=0x23b4 waiting on condition [0x00007f9de491d000]
  1888.    java.lang.Thread.State: WAITING (parking)
  1889.     at sun.misc.Unsafe.park(Native Method)
  1890.     - parking to wait for  <0x0000000149b65cc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1891.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1892.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1893.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1894.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  1895.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  1896.  
  1897. "RpcServer.Metadata.handler=4,queue=0,port=16020" #159 daemon prio=5 os_prio=0 tid=0x00007f9e374cd000 nid=0x23b3 waiting on condition [0x00007f9de4a1e000]
  1898.    java.lang.Thread.State: WAITING (parking)
  1899.     at sun.misc.Unsafe.park(Native Method)
  1900.     - parking to wait for  <0x0000000149b65cc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1901.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1902.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1903.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1904.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  1905.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  1906.  
  1907. "RpcServer.Metadata.handler=3,queue=0,port=16020" #158 daemon prio=5 os_prio=0 tid=0x00007f9e374cb000 nid=0x23b2 waiting on condition [0x00007f9de4b1f000]
  1908.    java.lang.Thread.State: WAITING (parking)
  1909.     at sun.misc.Unsafe.park(Native Method)
  1910.     - parking to wait for  <0x0000000149b65cc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1911.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1912.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1913.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1914.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  1915.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  1916.  
  1917. "RpcServer.Metadata.handler=2,queue=0,port=16020" #157 daemon prio=5 os_prio=0 tid=0x00007f9e374c9000 nid=0x23b1 waiting on condition [0x00007f9de4c20000]
  1918.    java.lang.Thread.State: WAITING (parking)
  1919.     at sun.misc.Unsafe.park(Native Method)
  1920.     - parking to wait for  <0x0000000149b65cc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1921.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1922.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1923.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1924.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  1925.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  1926.  
  1927. "RpcServer.Metadata.handler=1,queue=0,port=16020" #156 daemon prio=5 os_prio=0 tid=0x00007f9e374c6800 nid=0x23b0 waiting on condition [0x00007f9de4d21000]
  1928.    java.lang.Thread.State: WAITING (parking)
  1929.     at sun.misc.Unsafe.park(Native Method)
  1930.     - parking to wait for  <0x0000000149b65cc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1931.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1932.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1933.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1934.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  1935.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  1936.  
  1937. "RpcServer.Metadata.handler=0,queue=0,port=16020" #155 daemon prio=5 os_prio=0 tid=0x00007f9e374c4800 nid=0x23af waiting on condition [0x00007f9de4e22000]
  1938.    java.lang.Thread.State: WAITING (parking)
  1939.     at sun.misc.Unsafe.park(Native Method)
  1940.     - parking to wait for  <0x0000000149b65cc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1941.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1942.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1943.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1944.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  1945.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  1946.  
  1947. "RpcServer.Index.handler=29,queue=0,port=16020" #154 daemon prio=5 os_prio=0 tid=0x00007f9e374c2800 nid=0x23ae waiting on condition [0x00007f9de4f23000]
  1948.    java.lang.Thread.State: WAITING (parking)
  1949.     at sun.misc.Unsafe.park(Native Method)
  1950.     - parking to wait for  <0x0000000149b66818> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1951.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1952.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1953.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1954.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  1955.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  1956.  
  1957. "RpcServer.Index.handler=28,queue=0,port=16020" #153 daemon prio=5 os_prio=0 tid=0x00007f9e374c0800 nid=0x23ad waiting on condition [0x00007f9de5024000]
  1958.    java.lang.Thread.State: WAITING (parking)
  1959.     at sun.misc.Unsafe.park(Native Method)
  1960.     - parking to wait for  <0x0000000149b66818> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1961.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1962.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1963.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1964.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  1965.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  1966.  
  1967. "RpcServer.Index.handler=27,queue=0,port=16020" #152 daemon prio=5 os_prio=0 tid=0x00007f9e374be800 nid=0x23ac waiting on condition [0x00007f9de5125000]
  1968.    java.lang.Thread.State: WAITING (parking)
  1969.     at sun.misc.Unsafe.park(Native Method)
  1970.     - parking to wait for  <0x0000000149b66818> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1971.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1972.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1973.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1974.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  1975.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  1976.  
  1977. "RpcServer.Index.handler=26,queue=0,port=16020" #151 daemon prio=5 os_prio=0 tid=0x00007f9e374bc800 nid=0x23ab waiting on condition [0x00007f9de5226000]
  1978.    java.lang.Thread.State: WAITING (parking)
  1979.     at sun.misc.Unsafe.park(Native Method)
  1980.     - parking to wait for  <0x0000000149b66818> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1981.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1982.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1983.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1984.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  1985.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  1986.  
  1987. "RpcServer.Index.handler=25,queue=0,port=16020" #150 daemon prio=5 os_prio=0 tid=0x00007f9e374ba800 nid=0x23aa waiting on condition [0x00007f9de5327000]
  1988.    java.lang.Thread.State: WAITING (parking)
  1989.     at sun.misc.Unsafe.park(Native Method)
  1990.     - parking to wait for  <0x0000000149b66818> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1991.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1992.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1993.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1994.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  1995.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  1996.  
  1997. "RpcServer.Index.handler=24,queue=0,port=16020" #149 daemon prio=5 os_prio=0 tid=0x00007f9e374b8800 nid=0x23a9 waiting on condition [0x00007f9de5428000]
  1998.    java.lang.Thread.State: WAITING (parking)
  1999.     at sun.misc.Unsafe.park(Native Method)
  2000.     - parking to wait for  <0x0000000149b66818> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  2001.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  2002.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  2003.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  2004.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  2005.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2006.  
  2007. "RpcServer.Index.handler=23,queue=0,port=16020" #148 daemon prio=5 os_prio=0 tid=0x00007f9e374b6000 nid=0x23a8 waiting on condition [0x00007f9de5529000]
  2008.    java.lang.Thread.State: WAITING (parking)
  2009.     at sun.misc.Unsafe.park(Native Method)
  2010.     - parking to wait for  <0x0000000149b66818> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  2011.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  2012.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  2013.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  2014.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  2015.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2016.  
  2017. "RpcServer.Index.handler=22,queue=0,port=16020" #147 daemon prio=5 os_prio=0 tid=0x00007f9e374b4000 nid=0x23a7 waiting on condition [0x00007f9de562a000]
  2018.    java.lang.Thread.State: WAITING (parking)
  2019.     at sun.misc.Unsafe.park(Native Method)
  2020.     - parking to wait for  <0x0000000149b66818> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  2021.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  2022.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  2023.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  2024.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  2025.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2026.  
  2027. "RpcServer.Index.handler=21,queue=0,port=16020" #146 daemon prio=5 os_prio=0 tid=0x00007f9e374b1800 nid=0x23a6 waiting on condition [0x00007f9de572b000]
  2028.    java.lang.Thread.State: WAITING (parking)
  2029.     at sun.misc.Unsafe.park(Native Method)
  2030.     - parking to wait for  <0x0000000149b66818> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  2031.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  2032.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  2033.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  2034.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  2035.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2036.  
  2037. "RpcServer.Index.handler=20,queue=0,port=16020" #145 daemon prio=5 os_prio=0 tid=0x00007f9e374af800 nid=0x23a5 waiting on condition [0x00007f9de582c000]
  2038.    java.lang.Thread.State: WAITING (parking)
  2039.     at sun.misc.Unsafe.park(Native Method)
  2040.     - parking to wait for  <0x0000000149b66818> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  2041.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  2042.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  2043.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  2044.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  2045.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2046.  
  2047. "RpcServer.Index.handler=19,queue=0,port=16020" #144 daemon prio=5 os_prio=0 tid=0x00007f9e374ad800 nid=0x23a4 waiting on condition [0x00007f9de592d000]
  2048.    java.lang.Thread.State: WAITING (parking)
  2049.     at sun.misc.Unsafe.park(Native Method)
  2050.     - parking to wait for  <0x0000000149b66818> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  2051.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  2052.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  2053.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  2054.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  2055.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2056.  
  2057. "RpcServer.Index.handler=18,queue=0,port=16020" #143 daemon prio=5 os_prio=0 tid=0x00007f9e374ab800 nid=0x23a3 waiting on condition [0x00007f9de5a2e000]
  2058.    java.lang.Thread.State: WAITING (parking)
  2059.     at sun.misc.Unsafe.park(Native Method)
  2060.     - parking to wait for  <0x0000000149b66818> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  2061.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  2062.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  2063.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  2064.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  2065.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2066.  
  2067. "RpcServer.Index.handler=17,queue=0,port=16020" #142 daemon prio=5 os_prio=0 tid=0x00007f9e374a9800 nid=0x23a2 waiting on condition [0x00007f9de5b2f000]
  2068.    java.lang.Thread.State: WAITING (parking)
  2069.     at sun.misc.Unsafe.park(Native Method)
  2070.     - parking to wait for  <0x0000000149b66818> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  2071.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  2072.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  2073.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  2074.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  2075.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2076.  
  2077. "RpcServer.Index.handler=16,queue=0,port=16020" #141 daemon prio=5 os_prio=0 tid=0x00007f9e374a7000 nid=0x23a1 waiting on condition [0x00007f9de5c30000]
  2078.    java.lang.Thread.State: WAITING (parking)
  2079.     at sun.misc.Unsafe.park(Native Method)
  2080.     - parking to wait for  <0x0000000149b66818> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  2081.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  2082.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  2083.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  2084.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  2085.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2086.  
  2087. "RpcServer.Index.handler=15,queue=0,port=16020" #140 daemon prio=5 os_prio=0 tid=0x00007f9e374a5000 nid=0x23a0 waiting on condition [0x00007f9de5d31000]
  2088.    java.lang.Thread.State: WAITING (parking)
  2089.     at sun.misc.Unsafe.park(Native Method)
  2090.     - parking to wait for  <0x0000000149b66818> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  2091.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  2092.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  2093.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  2094.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  2095.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2096.  
  2097. "RpcServer.Index.handler=14,queue=0,port=16020" #139 daemon prio=5 os_prio=0 tid=0x00007f9e374a2800 nid=0x239f waiting on condition [0x00007f9de5e32000]
  2098.    java.lang.Thread.State: WAITING (parking)
  2099.     at sun.misc.Unsafe.park(Native Method)
  2100.     - parking to wait for  <0x0000000149b66818> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  2101.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  2102.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  2103.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  2104.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  2105.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2106.  
  2107. "RpcServer.Index.handler=13,queue=0,port=16020" #138 daemon prio=5 os_prio=0 tid=0x00007f9e374a0800 nid=0x239e waiting on condition [0x00007f9de5f33000]
  2108.    java.lang.Thread.State: WAITING (parking)
  2109.     at sun.misc.Unsafe.park(Native Method)
  2110.     - parking to wait for  <0x0000000149b66818> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  2111.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  2112.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  2113.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  2114.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  2115.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2116.  
  2117. "RpcServer.Index.handler=12,queue=0,port=16020" #137 daemon prio=5 os_prio=0 tid=0x00007f9e3749e800 nid=0x239d waiting on condition [0x00007f9de6034000]
  2118.    java.lang.Thread.State: WAITING (parking)
  2119.     at sun.misc.Unsafe.park(Native Method)
  2120.     - parking to wait for  <0x0000000149b66818> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  2121.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  2122.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  2123.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  2124.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  2125.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2126.  
  2127. "RpcServer.Index.handler=11,queue=0,port=16020" #136 daemon prio=5 os_prio=0 tid=0x00007f9e3749c800 nid=0x239c waiting on condition [0x00007f9de6135000]
  2128.    java.lang.Thread.State: WAITING (parking)
  2129.     at sun.misc.Unsafe.park(Native Method)
  2130.     - parking to wait for  <0x0000000149b66818> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  2131.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  2132.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  2133.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  2134.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  2135.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2136.  
  2137. "RpcServer.Index.handler=10,queue=0,port=16020" #135 daemon prio=5 os_prio=0 tid=0x00007f9e3749a800 nid=0x239b waiting on condition [0x00007f9de6236000]
  2138.    java.lang.Thread.State: WAITING (parking)
  2139.     at sun.misc.Unsafe.park(Native Method)
  2140.     - parking to wait for  <0x0000000149b66818> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  2141.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  2142.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  2143.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  2144.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  2145.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2146.  
  2147. "RpcServer.Index.handler=9,queue=0,port=16020" #134 daemon prio=5 os_prio=0 tid=0x00007f9e37498800 nid=0x239a waiting on condition [0x00007f9de6337000]
  2148.    java.lang.Thread.State: WAITING (parking)
  2149.     at sun.misc.Unsafe.park(Native Method)
  2150.     - parking to wait for  <0x0000000149b66818> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  2151.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  2152.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  2153.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  2154.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  2155.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2156.  
  2157. "RpcServer.Index.handler=8,queue=0,port=16020" #133 daemon prio=5 os_prio=0 tid=0x00007f9e37496000 nid=0x2399 waiting on condition [0x00007f9de6438000]
  2158.    java.lang.Thread.State: WAITING (parking)
  2159.     at sun.misc.Unsafe.park(Native Method)
  2160.     - parking to wait for  <0x0000000149b66818> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  2161.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  2162.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  2163.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  2164.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  2165.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2166.  
  2167. "RpcServer.Index.handler=7,queue=0,port=16020" #132 daemon prio=5 os_prio=0 tid=0x00007f9e37494000 nid=0x2398 waiting on condition [0x00007f9de6539000]
  2168.    java.lang.Thread.State: WAITING (parking)
  2169.     at sun.misc.Unsafe.park(Native Method)
  2170.     - parking to wait for  <0x0000000149b66818> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  2171.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  2172.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  2173.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  2174.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  2175.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2176.  
  2177. "RpcServer.Index.handler=6,queue=0,port=16020" #131 daemon prio=5 os_prio=0 tid=0x00007f9e37492800 nid=0x2397 waiting on condition [0x00007f9de663a000]
  2178.    java.lang.Thread.State: WAITING (parking)
  2179.     at sun.misc.Unsafe.park(Native Method)
  2180.     - parking to wait for  <0x0000000149b66818> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  2181.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  2182.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  2183.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  2184.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  2185.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2186.  
  2187. "RpcServer.Index.handler=5,queue=0,port=16020" #130 daemon prio=5 os_prio=0 tid=0x00007f9e37490000 nid=0x2396 waiting on condition [0x00007f9de673b000]
  2188.    java.lang.Thread.State: WAITING (parking)
  2189.     at sun.misc.Unsafe.park(Native Method)
  2190.     - parking to wait for  <0x0000000149b66818> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  2191.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  2192.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  2193.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  2194.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  2195.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2196.  
  2197. "RpcServer.Index.handler=4,queue=0,port=16020" #129 daemon prio=5 os_prio=0 tid=0x00007f9e3748e000 nid=0x2395 waiting on condition [0x00007f9de683c000]
  2198.    java.lang.Thread.State: WAITING (parking)
  2199.     at sun.misc.Unsafe.park(Native Method)
  2200.     - parking to wait for  <0x0000000149b66818> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  2201.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  2202.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  2203.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  2204.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  2205.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2206.  
  2207. "RpcServer.Index.handler=3,queue=0,port=16020" #128 daemon prio=5 os_prio=0 tid=0x00007f9e3748c000 nid=0x2394 waiting on condition [0x00007f9de693d000]
  2208.    java.lang.Thread.State: WAITING (parking)
  2209.     at sun.misc.Unsafe.park(Native Method)
  2210.     - parking to wait for  <0x0000000149b66818> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  2211.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  2212.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  2213.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  2214.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  2215.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2216.  
  2217. "RpcServer.Index.handler=2,queue=0,port=16020" #127 daemon prio=5 os_prio=0 tid=0x00007f9e3748a000 nid=0x2393 waiting on condition [0x00007f9de6a3e000]
  2218.    java.lang.Thread.State: WAITING (parking)
  2219.     at sun.misc.Unsafe.park(Native Method)
  2220.     - parking to wait for  <0x0000000149b66818> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  2221.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  2222.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  2223.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  2224.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  2225.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2226.  
  2227. "RpcServer.Index.handler=1,queue=0,port=16020" #126 daemon prio=5 os_prio=0 tid=0x00007f9e37487800 nid=0x2392 waiting on condition [0x00007f9de6b3f000]
  2228.    java.lang.Thread.State: WAITING (parking)
  2229.     at sun.misc.Unsafe.park(Native Method)
  2230.     - parking to wait for  <0x0000000149b66818> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  2231.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  2232.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  2233.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  2234.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  2235.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2236.  
  2237. "RpcServer.Index.handler=0,queue=0,port=16020" #125 daemon prio=5 os_prio=0 tid=0x00007f9e37485800 nid=0x2391 waiting on condition [0x00007f9de6c40000]
  2238.    java.lang.Thread.State: WAITING (parking)
  2239.     at sun.misc.Unsafe.park(Native Method)
  2240.     - parking to wait for  <0x0000000149b66818> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  2241.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  2242.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  2243.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  2244.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.getCallRunner(RpcExecutor.java:268)
  2245.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2246.  
  2247. "RpcServer.replication.FPBQ.Fifo.handler=2,queue=0,port=16020" #124 daemon prio=5 os_prio=0 tid=0x00007f9e37483800 nid=0x2390 waiting on condition [0x00007f9de6d41000]
  2248.    java.lang.Thread.State: WAITING (parking)
  2249.     at sun.misc.Unsafe.park(Native Method)
  2250.     - parking to wait for  <0x0000000150d8b168> (a java.util.concurrent.Semaphore$NonfairSync)
  2251.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  2252.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  2253.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
  2254.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
  2255.     at java.util.concurrent.Semaphore.acquire(Semaphore.java:312)
  2256.     at org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:103)
  2257.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2258.  
  2259. "RpcServer.replication.FPBQ.Fifo.handler=1,queue=0,port=16020" #123 daemon prio=5 os_prio=0 tid=0x00007f9e37481800 nid=0x238f waiting on condition [0x00007f9de6e42000]
  2260.    java.lang.Thread.State: WAITING (parking)
  2261.     at sun.misc.Unsafe.park(Native Method)
  2262.     - parking to wait for  <0x0000000150d90a28> (a java.util.concurrent.Semaphore$NonfairSync)
  2263.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  2264.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  2265.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
  2266.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
  2267.     at java.util.concurrent.Semaphore.acquire(Semaphore.java:312)
  2268.     at org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:103)
  2269.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2270.  
  2271. "RpcServer.replication.FPBQ.Fifo.handler=0,queue=0,port=16020" #122 daemon prio=5 os_prio=0 tid=0x00007f9e3747f800 nid=0x238e waiting on condition [0x00007f9de6f43000]
  2272.    java.lang.Thread.State: WAITING (parking)
  2273.     at sun.misc.Unsafe.park(Native Method)
  2274.     - parking to wait for  <0x0000000151350be0> (a java.util.concurrent.Semaphore$NonfairSync)
  2275.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  2276.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  2277.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
  2278.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
  2279.     at java.util.concurrent.Semaphore.acquire(Semaphore.java:312)
  2280.     at org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:103)
  2281.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2282.  
  2283. "RpcServer.priority.FPBQ.Fifo.handler=19,queue=1,port=16020" #121 daemon prio=5 os_prio=0 tid=0x00007f9e3747d800 nid=0x238d waiting on condition [0x00007f9de7044000]
  2284.    java.lang.Thread.State: WAITING (parking)
  2285.     at sun.misc.Unsafe.park(Native Method)
  2286.     - parking to wait for  <0x000000014ea3f2d8> (a java.util.concurrent.Semaphore$NonfairSync)
  2287.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  2288.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  2289.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
  2290.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
  2291.     at java.util.concurrent.Semaphore.acquire(Semaphore.java:312)
  2292.     at org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:103)
  2293.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2294.  
  2295. "RpcServer.priority.FPBQ.Fifo.handler=18,queue=0,port=16020" #120 daemon prio=5 os_prio=0 tid=0x00007f9e3747b000 nid=0x238c waiting on condition [0x00007f9de7145000]
  2296.    java.lang.Thread.State: WAITING (parking)
  2297.     at sun.misc.Unsafe.park(Native Method)
  2298.     - parking to wait for  <0x0000000150dab628> (a java.util.concurrent.Semaphore$NonfairSync)
  2299.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  2300.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  2301.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
  2302.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
  2303.     at java.util.concurrent.Semaphore.acquire(Semaphore.java:312)
  2304.     at org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:103)
  2305.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2306.  
  2307. "RpcServer.priority.FPBQ.Fifo.handler=17,queue=1,port=16020" #119 daemon prio=5 os_prio=0 tid=0x00007f9e37479000 nid=0x238b waiting on condition [0x00007f9de7246000]
  2308.    java.lang.Thread.State: WAITING (parking)
  2309.     at sun.misc.Unsafe.park(Native Method)
  2310.     - parking to wait for  <0x0000000150f6c9b8> (a java.util.concurrent.Semaphore$NonfairSync)
  2311.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  2312.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  2313.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
  2314.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
  2315.     at java.util.concurrent.Semaphore.acquire(Semaphore.java:312)
  2316.     at org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:103)
  2317.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2318.  
  2319. "RpcServer.priority.FPBQ.Fifo.handler=16,queue=0,port=16020" #118 daemon prio=5 os_prio=0 tid=0x00007f9e37477000 nid=0x238a waiting on condition [0x00007f9de7347000]
  2320.    java.lang.Thread.State: WAITING (parking)
  2321.     at sun.misc.Unsafe.park(Native Method)
  2322.     - parking to wait for  <0x0000000150d90ac8> (a java.util.concurrent.Semaphore$NonfairSync)
  2323.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  2324.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  2325.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
  2326.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
  2327.     at java.util.concurrent.Semaphore.acquire(Semaphore.java:312)
  2328.     at org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:103)
  2329.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2330.  
  2331. "RpcServer.priority.FPBQ.Fifo.handler=15,queue=1,port=16020" #117 daemon prio=5 os_prio=0 tid=0x00007f9e37474800 nid=0x2389 waiting on condition [0x00007f9de7448000]
  2332.    java.lang.Thread.State: WAITING (parking)
  2333.     at sun.misc.Unsafe.park(Native Method)
  2334.     - parking to wait for  <0x0000000150d8b208> (a java.util.concurrent.Semaphore$NonfairSync)
  2335.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  2336.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  2337.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
  2338.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
  2339.     at java.util.concurrent.Semaphore.acquire(Semaphore.java:312)
  2340.     at org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:103)
  2341.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2342.  
  2343. "RpcServer.priority.FPBQ.Fifo.handler=14,queue=0,port=16020" #116 daemon prio=5 os_prio=0 tid=0x00007f9e37472800 nid=0x2388 waiting on condition [0x00007f9de7549000]
  2344.    java.lang.Thread.State: WAITING (parking)
  2345.     at sun.misc.Unsafe.park(Native Method)
  2346.     - parking to wait for  <0x0000000150f6ca18> (a java.util.concurrent.Semaphore$NonfairSync)
  2347.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  2348.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  2349.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
  2350.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
  2351.     at java.util.concurrent.Semaphore.acquire(Semaphore.java:312)
  2352.     at org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:103)
  2353.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2354.  
  2355. "RpcServer.priority.FPBQ.Fifo.handler=13,queue=1,port=16020" #115 daemon prio=5 os_prio=0 tid=0x00007f9e37470800 nid=0x2387 waiting on condition [0x00007f9de764a000]
  2356.    java.lang.Thread.State: WAITING (parking)
  2357.     at sun.misc.Unsafe.park(Native Method)
  2358.     - parking to wait for  <0x0000000150d8b2a8> (a java.util.concurrent.Semaphore$NonfairSync)
  2359.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  2360.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  2361.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
  2362.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
  2363.     at java.util.concurrent.Semaphore.acquire(Semaphore.java:312)
  2364.     at org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:103)
  2365.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2366.  
  2367. "RpcServer.priority.FPBQ.Fifo.handler=12,queue=0,port=16020" #114 daemon prio=5 os_prio=0 tid=0x00007f9e3746e800 nid=0x2386 waiting on condition [0x00007f9de774b000]
  2368.    java.lang.Thread.State: WAITING (parking)
  2369.     at sun.misc.Unsafe.park(Native Method)
  2370.     - parking to wait for  <0x0000000150f6ca78> (a java.util.concurrent.Semaphore$NonfairSync)
  2371.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  2372.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  2373.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
  2374.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
  2375.     at java.util.concurrent.Semaphore.acquire(Semaphore.java:312)
  2376.     at org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:103)
  2377.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2378.  
  2379. "RpcServer.priority.FPBQ.Fifo.handler=11,queue=1,port=16020" #113 daemon prio=5 os_prio=0 tid=0x00007f9e3746c800 nid=0x2385 waiting on condition [0x00007f9de784c000]
  2380.    java.lang.Thread.State: WAITING (parking)
  2381.     at sun.misc.Unsafe.park(Native Method)
  2382.     - parking to wait for  <0x0000000150d8b348> (a java.util.concurrent.Semaphore$NonfairSync)
  2383.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  2384.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  2385.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
  2386.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
  2387.     at java.util.concurrent.Semaphore.acquire(Semaphore.java:312)
  2388.     at org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:103)
  2389.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2390.  
  2391. "RpcServer.priority.FPBQ.Fifo.handler=10,queue=0,port=16020" #112 daemon prio=5 os_prio=0 tid=0x00007f9e3746a800 nid=0x2384 waiting on condition [0x00007f9de794d000]
  2392.    java.lang.Thread.State: WAITING (parking)
  2393.     at sun.misc.Unsafe.park(Native Method)
  2394.     - parking to wait for  <0x0000000150d90b68> (a java.util.concurrent.Semaphore$NonfairSync)
  2395.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  2396.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  2397.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
  2398.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
  2399.     at java.util.concurrent.Semaphore.acquire(Semaphore.java:312)
  2400.     at org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:103)
  2401.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2402.  
  2403. "RpcServer.priority.FPBQ.Fifo.handler=9,queue=1,port=16020" #111 daemon prio=5 os_prio=0 tid=0x00007f9e37468800 nid=0x2383 waiting on condition [0x00007f9de7a4e000]
  2404.    java.lang.Thread.State: WAITING (parking)
  2405.     at sun.misc.Unsafe.park(Native Method)
  2406.     - parking to wait for  <0x0000000150d8b3e8> (a java.util.concurrent.Semaphore$NonfairSync)
  2407.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  2408.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  2409.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
  2410.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
  2411.     at java.util.concurrent.Semaphore.acquire(Semaphore.java:312)
  2412.     at org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:103)
  2413.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2414.  
  2415. "RpcServer.priority.FPBQ.Fifo.handler=8,queue=0,port=16020" #110 daemon prio=5 os_prio=0 tid=0x00007f9e37466800 nid=0x2382 waiting on condition [0x00007f9de7b4f000]
  2416.    java.lang.Thread.State: WAITING (parking)
  2417.     at sun.misc.Unsafe.park(Native Method)
  2418.     - parking to wait for  <0x0000000150d8b488> (a java.util.concurrent.Semaphore$NonfairSync)
  2419.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  2420.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  2421.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
  2422.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
  2423.     at java.util.concurrent.Semaphore.acquire(Semaphore.java:312)
  2424.     at org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:103)
  2425.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2426.  
  2427. "RpcServer.priority.FPBQ.Fifo.handler=7,queue=1,port=16020" #109 daemon prio=5 os_prio=0 tid=0x00007f9e37464800 nid=0x2381 waiting on condition [0x00007f9de7c50000]
  2428.    java.lang.Thread.State: WAITING (parking)
  2429.     at sun.misc.Unsafe.park(Native Method)
  2430.     - parking to wait for  <0x0000000150d90c08> (a java.util.concurrent.Semaphore$NonfairSync)
  2431.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  2432.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  2433.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
  2434.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
  2435.     at java.util.concurrent.Semaphore.acquire(Semaphore.java:312)
  2436.     at org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:103)
  2437.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2438.  
  2439. "RpcServer.priority.FPBQ.Fifo.handler=6,queue=0,port=16020" #108 daemon prio=5 os_prio=0 tid=0x00007f9e37462000 nid=0x2380 waiting on condition [0x00007f9de7d51000]
  2440.    java.lang.Thread.State: WAITING (parking)
  2441.     at sun.misc.Unsafe.park(Native Method)
  2442.     - parking to wait for  <0x0000000150d8b528> (a java.util.concurrent.Semaphore$NonfairSync)
  2443.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  2444.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  2445.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
  2446.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
  2447.     at java.util.concurrent.Semaphore.acquire(Semaphore.java:312)
  2448.     at org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:103)
  2449.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2450.  
  2451. "RpcServer.priority.FPBQ.Fifo.handler=5,queue=1,port=16020" #107 daemon prio=5 os_prio=0 tid=0x00007f9e37460000 nid=0x237f waiting on condition [0x00007f9de7e52000]
  2452.    java.lang.Thread.State: WAITING (parking)
  2453.     at sun.misc.Unsafe.park(Native Method)
  2454.     - parking to wait for  <0x0000000150d90ca8> (a java.util.concurrent.Semaphore$NonfairSync)
  2455.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  2456.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  2457.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
  2458.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
  2459.     at java.util.concurrent.Semaphore.acquire(Semaphore.java:312)
  2460.     at org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:103)
  2461.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2462.  
  2463. "RpcServer.priority.FPBQ.Fifo.handler=4,queue=0,port=16020" #106 daemon prio=5 os_prio=0 tid=0x00007f9e3745e000 nid=0x237e waiting on condition [0x00007f9de7f53000]
  2464.    java.lang.Thread.State: WAITING (parking)
  2465.     at sun.misc.Unsafe.park(Native Method)
  2466.     - parking to wait for  <0x0000000150d90d48> (a java.util.concurrent.Semaphore$NonfairSync)
  2467.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  2468.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  2469.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
  2470.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
  2471.     at java.util.concurrent.Semaphore.acquire(Semaphore.java:312)
  2472.     at org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:103)
  2473.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2474.  
  2475. "RpcServer.priority.FPBQ.Fifo.handler=3,queue=1,port=16020" #105 daemon prio=5 os_prio=0 tid=0x00007f9e3745c000 nid=0x237d waiting on condition [0x00007f9de8054000]
  2476.    java.lang.Thread.State: WAITING (parking)
  2477.     at sun.misc.Unsafe.park(Native Method)
  2478.     - parking to wait for  <0x0000000150d8b5c8> (a java.util.concurrent.Semaphore$NonfairSync)
  2479.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  2480.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  2481.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
  2482.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
  2483.     at java.util.concurrent.Semaphore.acquire(Semaphore.java:312)
  2484.     at org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:103)
  2485.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2486.  
  2487. "RpcServer.priority.FPBQ.Fifo.handler=2,queue=0,port=16020" #104 daemon prio=5 os_prio=0 tid=0x00007f9e3745a000 nid=0x237c waiting on condition [0x00007f9de8155000]
  2488.    java.lang.Thread.State: WAITING (parking)
  2489.     at sun.misc.Unsafe.park(Native Method)
  2490.     - parking to wait for  <0x0000000150d8b668> (a java.util.concurrent.Semaphore$NonfairSync)
  2491.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  2492.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  2493.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
  2494.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
  2495.     at java.util.concurrent.Semaphore.acquire(Semaphore.java:312)
  2496.     at org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:103)
  2497.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2498.  
  2499. "RpcServer.priority.FPBQ.Fifo.handler=1,queue=1,port=16020" #103 daemon prio=5 os_prio=0 tid=0x00007f9e37457800 nid=0x237b waiting on condition [0x00007f9de8256000]
  2500.    java.lang.Thread.State: WAITING (parking)
  2501.     at sun.misc.Unsafe.park(Native Method)
  2502.     - parking to wait for  <0x0000000150f6c858> (a java.util.concurrent.Semaphore$NonfairSync)
  2503.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  2504.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  2505.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
  2506.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
  2507.     at java.util.concurrent.Semaphore.acquire(Semaphore.java:312)
  2508.     at org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:103)
  2509.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2510.  
  2511. "RpcServer.priority.FPBQ.Fifo.handler=0,queue=0,port=16020" #102 daemon prio=5 os_prio=0 tid=0x00007f9e37455800 nid=0x237a waiting on condition [0x00007f9de8357000]
  2512.    java.lang.Thread.State: WAITING (parking)
  2513.     at sun.misc.Unsafe.park(Native Method)
  2514.     - parking to wait for  <0x0000000150d90de8> (a java.util.concurrent.Semaphore$NonfairSync)
  2515.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  2516.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  2517.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
  2518.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
  2519.     at java.util.concurrent.Semaphore.acquire(Semaphore.java:312)
  2520.     at org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:103)
  2521.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2522.  
  2523. "RpcServer.default.FPBQ.Fifo.handler=63,queue=3,port=16020" #101 daemon prio=5 os_prio=0 tid=0x00007f9e37453800 nid=0x2379 waiting for monitor entry [0x00007f9de8457000]
  2524.    java.lang.Thread.State: BLOCKED (on object monitor)
  2525.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  2526.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  2527.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  2528.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  2529.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  2530.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  2531.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  2532.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  2533.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  2534.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  2535.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  2536.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  2537.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  2538.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  2539.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  2540.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  2541.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  2542.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  2543.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  2544.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  2545.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  2546.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  2547.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  2548.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  2549.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  2550.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  2551.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  2552.     - locked <0x0000000124f5ac28> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  2553.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  2554.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  2555.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  2556.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  2557.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  2558.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  2559.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  2560.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  2561.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  2562.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2563.  
  2564. "RpcServer.default.FPBQ.Fifo.handler=62,queue=2,port=16020" #100 daemon prio=5 os_prio=0 tid=0x00007f9e37452000 nid=0x2378 waiting for monitor entry [0x00007f9de8558000]
  2565.    java.lang.Thread.State: BLOCKED (on object monitor)
  2566.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  2567.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  2568.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  2569.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  2570.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  2571.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  2572.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  2573.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  2574.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  2575.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  2576.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  2577.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  2578.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  2579.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  2580.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  2581.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  2582.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  2583.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  2584.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  2585.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  2586.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  2587.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  2588.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  2589.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  2590.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  2591.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  2592.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  2593.     - locked <0x00000001306f3a08> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  2594.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  2595.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  2596.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  2597.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  2598.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  2599.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  2600.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  2601.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  2602.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  2603.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2604.  
  2605. "RpcServer.default.FPBQ.Fifo.handler=61,queue=1,port=16020" #99 daemon prio=5 os_prio=0 tid=0x00007f9e37450000 nid=0x2377 waiting on condition [0x00007f9de865a000]
  2606.    java.lang.Thread.State: WAITING (parking)
  2607.     at sun.misc.Unsafe.park(Native Method)
  2608.     - parking to wait for  <0x0000000150d90e88> (a java.util.concurrent.Semaphore$NonfairSync)
  2609.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  2610.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  2611.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
  2612.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
  2613.     at java.util.concurrent.Semaphore.acquire(Semaphore.java:312)
  2614.     at org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:103)
  2615.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2616.  
  2617. "RpcServer.default.FPBQ.Fifo.handler=60,queue=0,port=16020" #98 daemon prio=5 os_prio=0 tid=0x00007f9e3744d800 nid=0x2376 waiting for monitor entry [0x00007f9de875a000]
  2618.    java.lang.Thread.State: BLOCKED (on object monitor)
  2619.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  2620.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  2621.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  2622.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  2623.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  2624.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  2625.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  2626.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  2627.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  2628.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  2629.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  2630.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  2631.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  2632.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  2633.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  2634.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  2635.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  2636.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  2637.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  2638.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  2639.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  2640.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  2641.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  2642.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  2643.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  2644.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  2645.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  2646.     - locked <0x0000000130004628> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  2647.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  2648.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  2649.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  2650.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  2651.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  2652.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  2653.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  2654.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  2655.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  2656.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2657.  
  2658. "RpcServer.default.FPBQ.Fifo.handler=59,queue=5,port=16020" #97 daemon prio=5 os_prio=0 tid=0x00007f9e3744b000 nid=0x2375 waiting for monitor entry [0x00007f9de885b000]
  2659.    java.lang.Thread.State: BLOCKED (on object monitor)
  2660.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  2661.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  2662.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  2663.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  2664.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  2665.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  2666.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  2667.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  2668.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  2669.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  2670.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  2671.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  2672.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  2673.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  2674.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  2675.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  2676.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  2677.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  2678.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  2679.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  2680.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  2681.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  2682.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  2683.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  2684.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  2685.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  2686.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  2687.     - locked <0x0000000130537a80> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  2688.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  2689.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  2690.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  2691.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  2692.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  2693.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  2694.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  2695.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  2696.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  2697.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2698.  
  2699. "RpcServer.default.FPBQ.Fifo.handler=58,queue=4,port=16020" #96 daemon prio=5 os_prio=0 tid=0x00007f9e37449000 nid=0x2374 waiting for monitor entry [0x00007f9de895c000]
  2700.    java.lang.Thread.State: BLOCKED (on object monitor)
  2701.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  2702.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  2703.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  2704.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  2705.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  2706.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  2707.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  2708.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  2709.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  2710.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  2711.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  2712.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  2713.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  2714.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  2715.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  2716.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  2717.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  2718.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  2719.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  2720.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  2721.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  2722.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  2723.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  2724.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  2725.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  2726.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  2727.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  2728.     - locked <0x00000001301caeb8> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  2729.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  2730.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  2731.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  2732.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  2733.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  2734.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  2735.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  2736.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  2737.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  2738.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2739.  
  2740. "RpcServer.default.FPBQ.Fifo.handler=57,queue=3,port=16020" #95 daemon prio=5 os_prio=0 tid=0x00007f9e37447000 nid=0x2373 waiting for monitor entry [0x00007f9de8a5d000]
  2741.    java.lang.Thread.State: BLOCKED (on object monitor)
  2742.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  2743.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  2744.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  2745.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  2746.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  2747.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  2748.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  2749.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  2750.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  2751.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  2752.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  2753.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  2754.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  2755.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  2756.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  2757.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  2758.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  2759.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  2760.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  2761.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  2762.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  2763.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  2764.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  2765.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  2766.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  2767.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  2768.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  2769.     - locked <0x000000012c0ba508> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  2770.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  2771.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  2772.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  2773.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  2774.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  2775.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  2776.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  2777.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  2778.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  2779.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2780.  
  2781. "RpcServer.default.FPBQ.Fifo.handler=56,queue=2,port=16020" #94 daemon prio=5 os_prio=0 tid=0x00007f9e37444800 nid=0x2372 waiting for monitor entry [0x00007f9de8b5e000]
  2782.    java.lang.Thread.State: BLOCKED (on object monitor)
  2783.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  2784.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  2785.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  2786.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  2787.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  2788.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  2789.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  2790.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  2791.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  2792.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  2793.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  2794.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  2795.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  2796.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  2797.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  2798.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  2799.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  2800.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  2801.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  2802.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  2803.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  2804.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  2805.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  2806.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  2807.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  2808.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  2809.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  2810.     - locked <0x00000001252d3f98> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  2811.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  2812.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  2813.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  2814.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  2815.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  2816.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  2817.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  2818.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  2819.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  2820.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2821.  
  2822. "RpcServer.default.FPBQ.Fifo.handler=55,queue=1,port=16020" #93 daemon prio=5 os_prio=0 tid=0x00007f9e37443000 nid=0x2371 waiting for monitor entry [0x00007f9de8c5f000]
  2823.    java.lang.Thread.State: BLOCKED (on object monitor)
  2824.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  2825.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  2826.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  2827.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  2828.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  2829.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  2830.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  2831.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  2832.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  2833.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  2834.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  2835.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  2836.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  2837.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  2838.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  2839.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  2840.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  2841.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  2842.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  2843.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  2844.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  2845.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  2846.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  2847.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  2848.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  2849.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  2850.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  2851.     - locked <0x00000001308a8378> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  2852.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  2853.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  2854.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  2855.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  2856.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  2857.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  2858.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  2859.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  2860.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  2861.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2862.  
  2863. "RpcServer.default.FPBQ.Fifo.handler=54,queue=0,port=16020" #92 daemon prio=5 os_prio=0 tid=0x00007f9e37440800 nid=0x2370 waiting for monitor entry [0x00007f9de8d60000]
  2864.    java.lang.Thread.State: BLOCKED (on object monitor)
  2865.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  2866.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  2867.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  2868.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  2869.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  2870.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  2871.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  2872.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  2873.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  2874.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  2875.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  2876.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  2877.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  2878.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  2879.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  2880.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  2881.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  2882.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  2883.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  2884.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  2885.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  2886.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  2887.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  2888.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  2889.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  2890.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  2891.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  2892.     - locked <0x000000012275ca80> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  2893.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  2894.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  2895.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  2896.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  2897.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  2898.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  2899.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  2900.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  2901.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  2902.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2903.  
  2904. "RpcServer.default.FPBQ.Fifo.handler=53,queue=5,port=16020" #91 daemon prio=5 os_prio=0 tid=0x00007f9e3743e800 nid=0x236f waiting for monitor entry [0x00007f9de8e61000]
  2905.    java.lang.Thread.State: BLOCKED (on object monitor)
  2906.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  2907.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  2908.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  2909.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  2910.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  2911.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  2912.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  2913.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  2914.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  2915.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  2916.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  2917.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  2918.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  2919.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  2920.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  2921.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  2922.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  2923.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  2924.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  2925.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  2926.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  2927.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  2928.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  2929.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  2930.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  2931.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  2932.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  2933.     - locked <0x000000012ea060e8> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  2934.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  2935.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  2936.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  2937.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  2938.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  2939.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  2940.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  2941.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  2942.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  2943.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2944.  
  2945. "RpcServer.default.FPBQ.Fifo.handler=52,queue=4,port=16020" #90 daemon prio=5 os_prio=0 tid=0x00007f9e3743c000 nid=0x236e waiting for monitor entry [0x00007f9de8f62000]
  2946.    java.lang.Thread.State: BLOCKED (on object monitor)
  2947.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  2948.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  2949.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  2950.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  2951.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  2952.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  2953.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  2954.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  2955.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  2956.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  2957.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  2958.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  2959.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  2960.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  2961.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  2962.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  2963.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  2964.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  2965.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  2966.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  2967.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  2968.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  2969.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  2970.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  2971.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  2972.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  2973.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  2974.     - locked <0x000000012d883fb8> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  2975.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  2976.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  2977.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  2978.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  2979.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  2980.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  2981.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  2982.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  2983.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  2984.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  2985.  
  2986. "RpcServer.default.FPBQ.Fifo.handler=51,queue=3,port=16020" #89 daemon prio=5 os_prio=0 tid=0x00007f9e3743a000 nid=0x236d waiting for monitor entry [0x00007f9de9063000]
  2987.    java.lang.Thread.State: BLOCKED (on object monitor)
  2988.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  2989.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  2990.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  2991.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  2992.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  2993.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  2994.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  2995.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  2996.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  2997.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  2998.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  2999.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  3000.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  3001.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  3002.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  3003.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  3004.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  3005.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  3006.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  3007.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  3008.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  3009.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  3010.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  3011.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  3012.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  3013.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  3014.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  3015.     - locked <0x0000000130a62a18> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  3016.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  3017.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  3018.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  3019.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  3020.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  3021.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  3022.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  3023.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  3024.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  3025.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  3026.  
  3027. "RpcServer.default.FPBQ.Fifo.handler=50,queue=2,port=16020" #88 daemon prio=5 os_prio=0 tid=0x00007f9e37438000 nid=0x236c waiting for monitor entry [0x00007f9de9164000]
  3028.    java.lang.Thread.State: BLOCKED (on object monitor)
  3029.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  3030.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  3031.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  3032.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  3033.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  3034.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  3035.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  3036.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  3037.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  3038.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  3039.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  3040.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  3041.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  3042.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  3043.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  3044.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  3045.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  3046.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  3047.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  3048.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  3049.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  3050.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  3051.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  3052.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  3053.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  3054.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  3055.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  3056.     - locked <0x00000001301cf618> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  3057.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  3058.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  3059.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  3060.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  3061.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  3062.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  3063.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  3064.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  3065.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  3066.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  3067.  
  3068. "RpcServer.default.FPBQ.Fifo.handler=49,queue=1,port=16020" #87 daemon prio=5 os_prio=0 tid=0x00007f9e37435800 nid=0x236b waiting for monitor entry [0x00007f9de9265000]
  3069.    java.lang.Thread.State: BLOCKED (on object monitor)
  3070.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  3071.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  3072.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  3073.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  3074.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  3075.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  3076.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  3077.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  3078.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  3079.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  3080.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  3081.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  3082.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  3083.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  3084.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  3085.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  3086.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  3087.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  3088.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  3089.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  3090.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  3091.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  3092.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  3093.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  3094.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  3095.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  3096.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  3097.     - locked <0x00000001306f8068> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  3098.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  3099.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  3100.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  3101.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  3102.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  3103.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  3104.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  3105.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  3106.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  3107.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  3108.  
  3109. "RpcServer.default.FPBQ.Fifo.handler=48,queue=0,port=16020" #86 daemon prio=5 os_prio=0 tid=0x00007f9e37433800 nid=0x236a waiting for monitor entry [0x00007f9de9366000]
  3110.    java.lang.Thread.State: BLOCKED (on object monitor)
  3111.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  3112.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  3113.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  3114.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  3115.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  3116.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  3117.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  3118.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  3119.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  3120.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  3121.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  3122.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  3123.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  3124.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  3125.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  3126.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  3127.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  3128.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  3129.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  3130.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  3131.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  3132.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  3133.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  3134.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  3135.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  3136.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  3137.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  3138.     - locked <0x00000001308aca58> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  3139.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  3140.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  3141.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  3142.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  3143.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  3144.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  3145.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  3146.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  3147.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  3148.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  3149.  
  3150. "RpcServer.default.FPBQ.Fifo.handler=47,queue=5,port=16020" #85 daemon prio=5 os_prio=0 tid=0x00007f9e37431800 nid=0x2369 waiting for monitor entry [0x00007f9de9467000]
  3151.    java.lang.Thread.State: BLOCKED (on object monitor)
  3152.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  3153.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  3154.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  3155.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  3156.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  3157.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  3158.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  3159.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  3160.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  3161.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  3162.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  3163.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  3164.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  3165.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  3166.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  3167.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  3168.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  3169.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  3170.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  3171.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  3172.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  3173.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  3174.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  3175.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  3176.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  3177.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  3178.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  3179.     - locked <0x0000000122944d10> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  3180.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  3181.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  3182.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  3183.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  3184.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  3185.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  3186.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  3187.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  3188.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  3189.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  3190.  
  3191. "RpcServer.default.FPBQ.Fifo.handler=46,queue=4,port=16020" #84 daemon prio=5 os_prio=0 tid=0x00007f9e3742f800 nid=0x2368 waiting for monitor entry [0x00007f9de9568000]
  3192.    java.lang.Thread.State: BLOCKED (on object monitor)
  3193.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  3194.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  3195.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  3196.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  3197.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  3198.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  3199.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  3200.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  3201.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  3202.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  3203.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  3204.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  3205.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  3206.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  3207.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  3208.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  3209.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  3210.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  3211.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  3212.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  3213.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  3214.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  3215.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  3216.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  3217.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  3218.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  3219.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  3220.     - locked <0x0000000130a67020> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  3221.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  3222.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  3223.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  3224.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  3225.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  3226.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  3227.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  3228.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  3229.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  3230.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  3231.  
  3232. "RpcServer.default.FPBQ.Fifo.handler=45,queue=3,port=16020" #83 daemon prio=5 os_prio=0 tid=0x00007f9e3742d800 nid=0x2367 waiting for monitor entry [0x00007f9de9669000]
  3233.    java.lang.Thread.State: BLOCKED (on object monitor)
  3234.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  3235.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  3236.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  3237.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  3238.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  3239.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  3240.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  3241.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  3242.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  3243.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  3244.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  3245.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  3246.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  3247.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  3248.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  3249.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  3250.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  3251.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  3252.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  3253.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  3254.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  3255.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  3256.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  3257.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  3258.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  3259.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  3260.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  3261.     - locked <0x0000000120718d98> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  3262.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  3263.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  3264.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  3265.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  3266.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  3267.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  3268.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  3269.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  3270.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  3271.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  3272.  
  3273. "RpcServer.default.FPBQ.Fifo.handler=44,queue=2,port=16020" #82 daemon prio=5 os_prio=0 tid=0x00007f9e3742b000 nid=0x2366 waiting for monitor entry [0x00007f9de976a000]
  3274.    java.lang.Thread.State: BLOCKED (on object monitor)
  3275.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  3276.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  3277.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  3278.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  3279.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  3280.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  3281.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  3282.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  3283.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  3284.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  3285.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  3286.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  3287.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  3288.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  3289.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  3290.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  3291.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  3292.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  3293.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  3294.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  3295.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  3296.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  3297.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  3298.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  3299.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  3300.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  3301.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  3302.     - locked <0x0000000120c26120> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  3303.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  3304.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  3305.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  3306.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  3307.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  3308.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  3309.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  3310.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  3311.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  3312.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  3313.  
  3314. "RpcServer.default.FPBQ.Fifo.handler=43,queue=1,port=16020" #81 daemon prio=5 os_prio=0 tid=0x00007f9e37429000 nid=0x2365 waiting for monitor entry [0x00007f9de986b000]
  3315.    java.lang.Thread.State: BLOCKED (on object monitor)
  3316.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  3317.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  3318.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  3319.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  3320.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  3321.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  3322.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  3323.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  3324.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  3325.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  3326.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  3327.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  3328.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  3329.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  3330.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  3331.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  3332.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  3333.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  3334.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  3335.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  3336.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  3337.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  3338.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  3339.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  3340.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  3341.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  3342.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  3343.     - locked <0x00000001308b1030> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  3344.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  3345.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  3346.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  3347.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  3348.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  3349.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  3350.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  3351.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  3352.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  3353.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  3354.  
  3355. "RpcServer.default.FPBQ.Fifo.handler=42,queue=0,port=16020" #80 daemon prio=5 os_prio=0 tid=0x00007f9e37427000 nid=0x2364 waiting for monitor entry [0x00007f9de996c000]
  3356.    java.lang.Thread.State: BLOCKED (on object monitor)
  3357.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  3358.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  3359.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  3360.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  3361.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  3362.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  3363.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  3364.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  3365.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  3366.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  3367.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  3368.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  3369.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  3370.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  3371.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  3372.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  3373.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  3374.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  3375.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  3376.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  3377.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  3378.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  3379.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  3380.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  3381.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  3382.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  3383.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  3384.     - locked <0x00000001216ed548> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  3385.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  3386.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  3387.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  3388.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  3389.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  3390.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  3391.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  3392.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  3393.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  3394.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  3395.  
  3396. "RpcServer.default.FPBQ.Fifo.handler=41,queue=5,port=16020" #79 daemon prio=5 os_prio=0 tid=0x00007f9e37425000 nid=0x2363 waiting for monitor entry [0x00007f9de9a6d000]
  3397.    java.lang.Thread.State: BLOCKED (on object monitor)
  3398.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  3399.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  3400.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  3401.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  3402.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  3403.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  3404.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  3405.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  3406.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  3407.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  3408.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  3409.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  3410.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  3411.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  3412.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  3413.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  3414.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  3415.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  3416.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  3417.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  3418.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  3419.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  3420.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  3421.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  3422.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  3423.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  3424.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  3425.     - locked <0x00000001235fa9e8> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  3426.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  3427.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  3428.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  3429.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  3430.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  3431.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  3432.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  3433.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  3434.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  3435.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  3436.  
  3437. "RpcServer.default.FPBQ.Fifo.handler=40,queue=4,port=16020" #78 daemon prio=5 os_prio=0 tid=0x00007f9e37423000 nid=0x2362 waiting for monitor entry [0x00007f9de9b6e000]
  3438.    java.lang.Thread.State: BLOCKED (on object monitor)
  3439.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  3440.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  3441.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  3442.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  3443.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  3444.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  3445.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  3446.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  3447.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  3448.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  3449.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  3450.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  3451.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  3452.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  3453.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  3454.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  3455.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  3456.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  3457.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  3458.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  3459.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  3460.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  3461.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  3462.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  3463.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  3464.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  3465.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  3466.     - locked <0x00000001223480c8> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  3467.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  3468.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  3469.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  3470.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  3471.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  3472.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  3473.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  3474.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  3475.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  3476.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  3477.  
  3478. "RpcServer.default.FPBQ.Fifo.handler=39,queue=3,port=16020" #77 daemon prio=5 os_prio=0 tid=0x00007f9e37421000 nid=0x2361 waiting for monitor entry [0x00007f9de9c6f000]
  3479.    java.lang.Thread.State: BLOCKED (on object monitor)
  3480.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  3481.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  3482.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  3483.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  3484.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  3485.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  3486.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  3487.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  3488.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  3489.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  3490.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  3491.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  3492.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  3493.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  3494.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  3495.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  3496.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  3497.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  3498.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  3499.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  3500.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  3501.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  3502.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  3503.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  3504.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  3505.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  3506.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  3507.     - locked <0x00000001245be9d0> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  3508.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  3509.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  3510.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  3511.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  3512.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  3513.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  3514.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  3515.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  3516.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  3517.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  3518.  
  3519. "RpcServer.default.FPBQ.Fifo.handler=38,queue=2,port=16020" #76 daemon prio=5 os_prio=0 tid=0x00007f9e3741e800 nid=0x2360 waiting for monitor entry [0x00007f9de9d70000]
  3520.    java.lang.Thread.State: BLOCKED (on object monitor)
  3521.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  3522.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  3523.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  3524.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  3525.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  3526.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  3527.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  3528.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  3529.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  3530.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  3531.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  3532.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  3533.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  3534.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  3535.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  3536.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  3537.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  3538.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  3539.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  3540.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  3541.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  3542.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  3543.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  3544.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  3545.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  3546.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  3547.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  3548.     - locked <0x0000000130a6b6a8> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  3549.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  3550.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  3551.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  3552.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  3553.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  3554.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  3555.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  3556.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  3557.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  3558.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  3559.  
  3560. "RpcServer.default.FPBQ.Fifo.handler=37,queue=1,port=16020" #75 daemon prio=5 os_prio=0 tid=0x00007f9e3741c800 nid=0x235f waiting on condition [0x00007f9de9e72000]
  3561.    java.lang.Thread.State: WAITING (parking)
  3562.     at sun.misc.Unsafe.park(Native Method)
  3563.     - parking to wait for  <0x0000000150da3808> (a java.util.concurrent.Semaphore$NonfairSync)
  3564.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  3565.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  3566.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
  3567.     at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
  3568.     at java.util.concurrent.Semaphore.acquire(Semaphore.java:312)
  3569.     at org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:103)
  3570.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  3571.  
  3572. "RpcServer.default.FPBQ.Fifo.handler=36,queue=0,port=16020" #74 daemon prio=5 os_prio=0 tid=0x00007f9e3741a800 nid=0x235e waiting for monitor entry [0x00007f9de9f72000]
  3573.    java.lang.Thread.State: BLOCKED (on object monitor)
  3574.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  3575.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  3576.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  3577.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  3578.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  3579.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  3580.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  3581.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  3582.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  3583.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  3584.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  3585.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  3586.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  3587.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  3588.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  3589.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  3590.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  3591.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  3592.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  3593.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  3594.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  3595.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  3596.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  3597.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  3598.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  3599.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  3600.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  3601.     - locked <0x00000001308b5578> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  3602.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  3603.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  3604.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  3605.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  3606.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  3607.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  3608.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  3609.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  3610.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  3611.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  3612.  
  3613. "RpcServer.default.FPBQ.Fifo.handler=35,queue=5,port=16020" #73 daemon prio=5 os_prio=0 tid=0x00007f9e37418800 nid=0x235d waiting for monitor entry [0x00007f9dea073000]
  3614.    java.lang.Thread.State: BLOCKED (on object monitor)
  3615.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  3616.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  3617.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  3618.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  3619.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  3620.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  3621.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  3622.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  3623.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  3624.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  3625.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  3626.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  3627.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  3628.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  3629.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  3630.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  3631.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  3632.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  3633.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  3634.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  3635.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  3636.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  3637.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  3638.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  3639.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  3640.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  3641.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  3642.     - locked <0x00000001301d4688> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  3643.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  3644.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  3645.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  3646.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  3647.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  3648.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  3649.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  3650.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  3651.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  3652.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  3653.  
  3654. "RpcServer.default.FPBQ.Fifo.handler=34,queue=4,port=16020" #72 daemon prio=5 os_prio=0 tid=0x00007f9e37416000 nid=0x235c waiting for monitor entry [0x00007f9dea174000]
  3655.    java.lang.Thread.State: BLOCKED (on object monitor)
  3656.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  3657.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  3658.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  3659.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  3660.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  3661.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  3662.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  3663.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  3664.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  3665.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  3666.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  3667.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  3668.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  3669.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  3670.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  3671.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  3672.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  3673.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  3674.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  3675.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  3676.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  3677.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  3678.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  3679.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  3680.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  3681.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  3682.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  3683.     - locked <0x0000000130a70610> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  3684.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  3685.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  3686.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  3687.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  3688.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  3689.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  3690.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  3691.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  3692.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  3693.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  3694.  
  3695. "RpcServer.default.FPBQ.Fifo.handler=33,queue=3,port=16020" #71 daemon prio=5 os_prio=0 tid=0x00007f9e37414000 nid=0x235b waiting for monitor entry [0x00007f9dea275000]
  3696.    java.lang.Thread.State: BLOCKED (on object monitor)
  3697.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  3698.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  3699.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  3700.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  3701.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  3702.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  3703.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  3704.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  3705.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  3706.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  3707.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  3708.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  3709.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  3710.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  3711.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  3712.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  3713.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  3714.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  3715.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  3716.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  3717.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  3718.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  3719.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  3720.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  3721.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  3722.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  3723.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  3724.     - locked <0x00000001308baca0> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  3725.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  3726.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  3727.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  3728.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  3729.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  3730.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  3731.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  3732.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  3733.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  3734.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  3735.  
  3736. "RpcServer.default.FPBQ.Fifo.handler=32,queue=2,port=16020" #70 daemon prio=5 os_prio=0 tid=0x00007f9e37412000 nid=0x235a waiting for monitor entry [0x00007f9dea376000]
  3737.    java.lang.Thread.State: BLOCKED (on object monitor)
  3738.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  3739.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  3740.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  3741.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  3742.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  3743.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  3744.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  3745.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  3746.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  3747.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  3748.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  3749.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  3750.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  3751.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  3752.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  3753.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  3754.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  3755.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  3756.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  3757.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  3758.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  3759.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  3760.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  3761.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  3762.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  3763.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  3764.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  3765.     - locked <0x0000000130701700> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  3766.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  3767.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  3768.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  3769.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  3770.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  3771.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  3772.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  3773.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  3774.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  3775.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  3776.  
  3777. "RpcServer.default.FPBQ.Fifo.handler=31,queue=1,port=16020" #69 daemon prio=5 os_prio=0 tid=0x00007f9e37410000 nid=0x2359 waiting for monitor entry [0x00007f9dea477000]
  3778.    java.lang.Thread.State: BLOCKED (on object monitor)
  3779.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  3780.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  3781.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  3782.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  3783.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  3784.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  3785.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  3786.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  3787.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  3788.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  3789.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  3790.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  3791.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  3792.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  3793.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  3794.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  3795.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  3796.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  3797.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  3798.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  3799.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  3800.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  3801.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  3802.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  3803.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  3804.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  3805.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  3806.     - locked <0x000000012bd1e560> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  3807.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  3808.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  3809.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  3810.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  3811.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  3812.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  3813.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  3814.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  3815.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  3816.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  3817.  
  3818. "RpcServer.default.FPBQ.Fifo.handler=30,queue=0,port=16020" #68 daemon prio=5 os_prio=0 tid=0x00007f9e3740e000 nid=0x2358 in Object.wait() [0x00007f9dea579000]
  3819.    java.lang.Thread.State: WAITING (on object monitor)
  3820.     at java.lang.Object.wait(Native Method)
  3821.     at java.lang.Object.wait(Object.java:502)
  3822.     at org.apache.hadoop.hbase.util.IdLock.getLockEntry(IdLock.java:81)
  3823.     - locked <0x0000000130a70b98> (a org.apache.hadoop.hbase.util.IdLock$Entry)
  3824.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:416)
  3825.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  3826.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  3827.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  3828.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  3829.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  3830.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  3831.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  3832.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  3833.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  3834.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  3835.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  3836.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  3837.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  3838.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  3839.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  3840.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  3841.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  3842.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  3843.     - locked <0x0000000130a71858> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  3844.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  3845.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  3846.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  3847.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  3848.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  3849.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  3850.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  3851.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  3852.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  3853.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  3854.  
  3855. "RpcServer.default.FPBQ.Fifo.handler=29,queue=5,port=16020" #67 daemon prio=5 os_prio=0 tid=0x00007f9e3740b800 nid=0x2357 waiting for monitor entry [0x00007f9dea679000]
  3856.    java.lang.Thread.State: BLOCKED (on object monitor)
  3857.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  3858.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  3859.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  3860.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  3861.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  3862.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  3863.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  3864.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  3865.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  3866.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  3867.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  3868.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  3869.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  3870.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  3871.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  3872.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  3873.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  3874.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  3875.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  3876.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  3877.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  3878.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  3879.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  3880.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  3881.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  3882.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  3883.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  3884.     - locked <0x00000001308c00d0> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  3885.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  3886.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  3887.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  3888.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  3889.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  3890.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  3891.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  3892.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  3893.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  3894.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  3895.  
  3896. "RpcServer.default.FPBQ.Fifo.handler=28,queue=4,port=16020" #66 daemon prio=5 os_prio=0 tid=0x00007f9e37409800 nid=0x2356 waiting for monitor entry [0x00007f9dea77a000]
  3897.    java.lang.Thread.State: BLOCKED (on object monitor)
  3898.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  3899.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  3900.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  3901.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  3902.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  3903.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  3904.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  3905.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  3906.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  3907.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  3908.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  3909.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  3910.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  3911.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  3912.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  3913.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  3914.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  3915.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  3916.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  3917.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  3918.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  3919.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  3920.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  3921.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  3922.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  3923.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  3924.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  3925.     - locked <0x0000000130008e28> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  3926.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  3927.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  3928.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  3929.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  3930.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  3931.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  3932.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  3933.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  3934.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  3935.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  3936.  
  3937. "RpcServer.default.FPBQ.Fifo.handler=27,queue=3,port=16020" #65 daemon prio=5 os_prio=0 tid=0x00007f9e37407800 nid=0x2355 waiting for monitor entry [0x00007f9dea87b000]
  3938.    java.lang.Thread.State: BLOCKED (on object monitor)
  3939.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  3940.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  3941.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  3942.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  3943.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  3944.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  3945.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  3946.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  3947.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  3948.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  3949.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  3950.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  3951.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  3952.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  3953.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  3954.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  3955.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  3956.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  3957.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  3958.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  3959.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  3960.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  3961.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  3962.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  3963.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  3964.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  3965.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  3966.     - locked <0x00000001301d9738> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  3967.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  3968.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  3969.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  3970.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  3971.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  3972.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  3973.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  3974.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  3975.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  3976.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  3977.  
  3978. "RpcServer.default.FPBQ.Fifo.handler=26,queue=2,port=16020" #64 daemon prio=5 os_prio=0 tid=0x00007f9e37405800 nid=0x2354 waiting for monitor entry [0x00007f9dea97c000]
  3979.    java.lang.Thread.State: BLOCKED (on object monitor)
  3980.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  3981.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  3982.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  3983.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  3984.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  3985.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  3986.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  3987.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  3988.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  3989.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  3990.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  3991.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  3992.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  3993.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  3994.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  3995.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  3996.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  3997.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  3998.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  3999.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  4000.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  4001.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  4002.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  4003.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  4004.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  4005.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  4006.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  4007.     - locked <0x00000001220815f8> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  4008.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  4009.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  4010.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  4011.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  4012.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  4013.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  4014.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  4015.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  4016.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  4017.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  4018.  
  4019. "RpcServer.default.FPBQ.Fifo.handler=25,queue=1,port=16020" #63 daemon prio=5 os_prio=0 tid=0x00007f9e37403800 nid=0x2353 waiting for monitor entry [0x00007f9deaa7d000]
  4020.    java.lang.Thread.State: BLOCKED (on object monitor)
  4021.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  4022.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  4023.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  4024.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  4025.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  4026.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  4027.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  4028.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  4029.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  4030.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  4031.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  4032.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  4033.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  4034.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  4035.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  4036.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  4037.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  4038.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  4039.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  4040.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  4041.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  4042.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  4043.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  4044.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  4045.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  4046.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  4047.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  4048.     - locked <0x000000012b7dfdf0> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  4049.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  4050.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  4051.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  4052.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  4053.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  4054.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  4055.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  4056.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  4057.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  4058.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  4059.  
  4060. "RpcServer.default.FPBQ.Fifo.handler=24,queue=0,port=16020" #62 daemon prio=5 os_prio=0 tid=0x00007f9e37401800 nid=0x2352 waiting for monitor entry [0x00007f9deab7e000]
  4061.    java.lang.Thread.State: BLOCKED (on object monitor)
  4062.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  4063.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  4064.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  4065.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  4066.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  4067.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  4068.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  4069.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  4070.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  4071.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  4072.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  4073.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  4074.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  4075.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  4076.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  4077.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  4078.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  4079.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  4080.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  4081.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  4082.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  4083.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  4084.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  4085.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  4086.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  4087.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  4088.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  4089.     - locked <0x00000001308c5538> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  4090.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  4091.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  4092.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  4093.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  4094.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  4095.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  4096.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  4097.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  4098.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  4099.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  4100.  
  4101. "RpcServer.default.FPBQ.Fifo.handler=23,queue=5,port=16020" #61 daemon prio=5 os_prio=0 tid=0x00007f9e373ff000 nid=0x2351 waiting for monitor entry [0x00007f9deac7f000]
  4102.    java.lang.Thread.State: BLOCKED (on object monitor)
  4103.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  4104.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  4105.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  4106.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  4107.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  4108.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  4109.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  4110.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  4111.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  4112.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  4113.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  4114.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  4115.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  4116.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  4117.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  4118.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  4119.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  4120.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  4121.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  4122.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  4123.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  4124.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  4125.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  4126.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  4127.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  4128.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  4129.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  4130.     - locked <0x0000000130a76d28> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  4131.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  4132.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  4133.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  4134.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  4135.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  4136.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  4137.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  4138.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  4139.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  4140.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  4141.  
  4142. "RpcServer.default.FPBQ.Fifo.handler=22,queue=4,port=16020" #60 daemon prio=5 os_prio=0 tid=0x00007f9e373fd000 nid=0x2350 waiting for monitor entry [0x00007f9dead80000]
  4143.    java.lang.Thread.State: BLOCKED (on object monitor)
  4144.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  4145.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  4146.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  4147.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  4148.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  4149.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  4150.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  4151.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  4152.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  4153.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  4154.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  4155.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  4156.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  4157.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  4158.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  4159.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  4160.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  4161.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  4162.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  4163.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  4164.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  4165.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  4166.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  4167.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  4168.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  4169.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  4170.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  4171.     - locked <0x000000013000dea0> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  4172.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  4173.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  4174.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  4175.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  4176.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  4177.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  4178.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  4179.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  4180.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  4181.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  4182.  
  4183. "RpcServer.default.FPBQ.Fifo.handler=21,queue=3,port=16020" #59 daemon prio=5 os_prio=0 tid=0x00007f9e373fb000 nid=0x234f waiting for monitor entry [0x00007f9deae81000]
  4184.    java.lang.Thread.State: BLOCKED (on object monitor)
  4185.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  4186.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  4187.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  4188.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  4189.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  4190.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  4191.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  4192.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  4193.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  4194.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  4195.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  4196.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  4197.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  4198.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  4199.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  4200.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  4201.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  4202.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  4203.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  4204.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  4205.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  4206.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  4207.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  4208.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  4209.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  4210.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  4211.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  4212.     - locked <0x0000000181d0fc78> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  4213.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  4214.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  4215.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  4216.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  4217.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  4218.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  4219.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  4220.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  4221.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  4222.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  4223.  
  4224. "RpcServer.default.FPBQ.Fifo.handler=20,queue=2,port=16020" #58 daemon prio=5 os_prio=0 tid=0x00007f9e373f9000 nid=0x234e waiting for monitor entry [0x00007f9deaf82000]
  4225.    java.lang.Thread.State: BLOCKED (on object monitor)
  4226.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  4227.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  4228.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  4229.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  4230.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  4231.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  4232.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  4233.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  4234.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  4235.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  4236.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  4237.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  4238.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  4239.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  4240.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  4241.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  4242.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  4243.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  4244.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  4245.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  4246.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  4247.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  4248.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  4249.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  4250.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  4251.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  4252.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  4253.     - locked <0x0000000130706a88> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  4254.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  4255.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  4256.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  4257.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  4258.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  4259.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  4260.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  4261.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  4262.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  4263.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  4264.  
  4265. "RpcServer.default.FPBQ.Fifo.handler=19,queue=1,port=16020" #57 daemon prio=5 os_prio=0 tid=0x00007f9e373f7000 nid=0x234d waiting for monitor entry [0x00007f9deb083000]
  4266.    java.lang.Thread.State: BLOCKED (on object monitor)
  4267.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  4268.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  4269.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  4270.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  4271.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  4272.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  4273.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  4274.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  4275.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  4276.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  4277.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  4278.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  4279.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  4280.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  4281.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  4282.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  4283.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  4284.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  4285.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  4286.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  4287.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  4288.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  4289.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  4290.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  4291.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  4292.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  4293.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  4294.     - locked <0x000000012c795d48> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  4295.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  4296.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  4297.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  4298.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  4299.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  4300.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  4301.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  4302.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  4303.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  4304.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  4305.  
  4306. "RpcServer.default.FPBQ.Fifo.handler=18,queue=0,port=16020" #56 daemon prio=5 os_prio=0 tid=0x00007f9e373f4800 nid=0x234c waiting for monitor entry [0x00007f9deb184000]
  4307.    java.lang.Thread.State: BLOCKED (on object monitor)
  4308.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  4309.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  4310.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  4311.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  4312.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  4313.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  4314.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  4315.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  4316.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  4317.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  4318.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  4319.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  4320.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  4321.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  4322.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  4323.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  4324.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  4325.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  4326.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  4327.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  4328.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  4329.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  4330.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  4331.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  4332.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  4333.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  4334.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  4335.     - locked <0x00000001308ca9e0> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  4336.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  4337.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  4338.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  4339.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  4340.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  4341.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  4342.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  4343.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  4344.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  4345.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  4346.  
  4347. "RpcServer.default.FPBQ.Fifo.handler=17,queue=5,port=16020" #55 daemon prio=5 os_prio=0 tid=0x00007f9e373f2800 nid=0x234b waiting for monitor entry [0x00007f9deb285000]
  4348.    java.lang.Thread.State: BLOCKED (on object monitor)
  4349.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  4350.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  4351.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  4352.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  4353.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  4354.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  4355.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  4356.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  4357.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  4358.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  4359.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  4360.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  4361.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  4362.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  4363.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  4364.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  4365.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  4366.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  4367.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  4368.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  4369.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  4370.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  4371.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  4372.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  4373.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  4374.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  4375.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  4376.     - locked <0x00000001247db180> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  4377.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  4378.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  4379.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  4380.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  4381.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  4382.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  4383.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  4384.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  4385.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  4386.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  4387.  
  4388. "RpcServer.default.FPBQ.Fifo.handler=16,queue=4,port=16020" #54 daemon prio=5 os_prio=0 tid=0x00007f9e373f0800 nid=0x234a waiting for monitor entry [0x00007f9deb386000]
  4389.    java.lang.Thread.State: BLOCKED (on object monitor)
  4390.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  4391.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  4392.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  4393.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  4394.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  4395.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  4396.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  4397.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  4398.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  4399.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  4400.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  4401.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  4402.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  4403.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  4404.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  4405.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  4406.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  4407.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  4408.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  4409.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  4410.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  4411.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  4412.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  4413.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  4414.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  4415.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  4416.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  4417.     - locked <0x0000000130a7c188> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  4418.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  4419.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  4420.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  4421.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  4422.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  4423.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  4424.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  4425.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  4426.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  4427.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  4428.  
  4429. "RpcServer.default.FPBQ.Fifo.handler=15,queue=3,port=16020" #53 daemon prio=5 os_prio=0 tid=0x00007f9e373ee800 nid=0x2349 waiting for monitor entry [0x00007f9deb487000]
  4430.    java.lang.Thread.State: BLOCKED (on object monitor)
  4431.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  4432.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  4433.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  4434.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  4435.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  4436.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  4437.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  4438.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  4439.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  4440.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  4441.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  4442.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  4443.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  4444.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  4445.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  4446.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  4447.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  4448.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  4449.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  4450.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  4451.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  4452.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  4453.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  4454.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  4455.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  4456.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  4457.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  4458.     - locked <0x000000012ea4e8e0> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  4459.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  4460.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  4461.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  4462.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  4463.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  4464.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  4465.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  4466.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  4467.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  4468.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  4469.  
  4470. "RpcServer.default.FPBQ.Fifo.handler=14,queue=2,port=16020" #52 daemon prio=5 os_prio=0 tid=0x00007f9e373ec800 nid=0x2348 waiting for monitor entry [0x00007f9deb588000]
  4471.    java.lang.Thread.State: BLOCKED (on object monitor)
  4472.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  4473.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  4474.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  4475.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  4476.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  4477.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  4478.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  4479.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  4480.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  4481.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  4482.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  4483.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  4484.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  4485.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  4486.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  4487.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  4488.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  4489.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  4490.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  4491.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  4492.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  4493.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  4494.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  4495.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  4496.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  4497.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  4498.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  4499.     - locked <0x000000012db84268> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  4500.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  4501.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  4502.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  4503.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  4504.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  4505.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  4506.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  4507.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  4508.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  4509.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  4510.  
  4511. "RpcServer.default.FPBQ.Fifo.handler=13,queue=1,port=16020" #51 daemon prio=5 os_prio=0 tid=0x00007f9e373ea000 nid=0x2347 waiting for monitor entry [0x00007f9deb689000]
  4512.    java.lang.Thread.State: BLOCKED (on object monitor)
  4513.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  4514.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  4515.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  4516.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  4517.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  4518.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  4519.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  4520.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  4521.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  4522.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  4523.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  4524.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  4525.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  4526.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  4527.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  4528.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  4529.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  4530.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  4531.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  4532.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  4533.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  4534.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  4535.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  4536.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  4537.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  4538.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  4539.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  4540.     - locked <0x00000001301debc8> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  4541.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  4542.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  4543.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  4544.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  4545.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  4546.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  4547.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  4548.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  4549.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  4550.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  4551.  
  4552. "RpcServer.default.FPBQ.Fifo.handler=12,queue=0,port=16020" #50 daemon prio=5 os_prio=0 tid=0x00007f9e373e8000 nid=0x2346 waiting for monitor entry [0x00007f9deb78a000]
  4553.    java.lang.Thread.State: BLOCKED (on object monitor)
  4554.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  4555.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  4556.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  4557.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  4558.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  4559.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  4560.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  4561.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  4562.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  4563.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  4564.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  4565.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  4566.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  4567.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  4568.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  4569.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  4570.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  4571.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  4572.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  4573.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  4574.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  4575.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  4576.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  4577.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  4578.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  4579.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  4580.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  4581.     - locked <0x000000013070bfb0> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  4582.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  4583.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  4584.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  4585.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  4586.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  4587.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  4588.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  4589.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  4590.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  4591.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  4592.  
  4593. "RpcServer.default.FPBQ.Fifo.handler=11,queue=5,port=16020" #49 daemon prio=5 os_prio=0 tid=0x00007f9e373e6000 nid=0x2345 waiting for monitor entry [0x00007f9deb88b000]
  4594.    java.lang.Thread.State: BLOCKED (on object monitor)
  4595.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  4596.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  4597.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  4598.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  4599.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  4600.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  4601.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  4602.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  4603.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  4604.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  4605.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  4606.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  4607.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  4608.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  4609.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  4610.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  4611.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  4612.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  4613.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  4614.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  4615.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  4616.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  4617.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  4618.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  4619.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  4620.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  4621.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  4622.     - locked <0x0000000130013348> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  4623.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  4624.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  4625.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  4626.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  4627.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  4628.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  4629.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  4630.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  4631.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  4632.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  4633.  
  4634. "RpcServer.default.FPBQ.Fifo.handler=10,queue=4,port=16020" #48 daemon prio=5 os_prio=0 tid=0x00007f9e373e4000 nid=0x2344 waiting for monitor entry [0x00007f9deb98c000]
  4635.    java.lang.Thread.State: BLOCKED (on object monitor)
  4636.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  4637.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  4638.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  4639.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  4640.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  4641.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  4642.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  4643.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  4644.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  4645.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  4646.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  4647.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  4648.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  4649.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  4650.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  4651.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  4652.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  4653.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  4654.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  4655.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  4656.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  4657.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  4658.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  4659.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  4660.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  4661.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  4662.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  4663.     - locked <0x0000000130a81578> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  4664.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  4665.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  4666.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  4667.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  4668.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  4669.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  4670.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  4671.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  4672.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  4673.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  4674.  
  4675. "RpcServer.default.FPBQ.Fifo.handler=9,queue=3,port=16020" #47 daemon prio=5 os_prio=0 tid=0x00007f9e373e2000 nid=0x2343 waiting for monitor entry [0x00007f9deba8d000]
  4676.    java.lang.Thread.State: BLOCKED (on object monitor)
  4677.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  4678.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  4679.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  4680.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  4681.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  4682.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  4683.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  4684.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  4685.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  4686.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  4687.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  4688.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  4689.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  4690.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  4691.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  4692.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  4693.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  4694.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  4695.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  4696.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  4697.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  4698.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  4699.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  4700.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  4701.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  4702.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  4703.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  4704.     - locked <0x00000001308cff20> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  4705.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  4706.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  4707.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  4708.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  4709.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  4710.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  4711.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  4712.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  4713.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  4714.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  4715.  
  4716. "RpcServer.default.FPBQ.Fifo.handler=8,queue=2,port=16020" #46 daemon prio=5 os_prio=0 tid=0x00007f9e373e0000 nid=0x2342 waiting for monitor entry [0x00007f9debb8e000]
  4717.    java.lang.Thread.State: BLOCKED (on object monitor)
  4718.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  4719.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  4720.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  4721.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  4722.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  4723.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  4724.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  4725.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  4726.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  4727.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  4728.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  4729.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  4730.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  4731.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  4732.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  4733.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  4734.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  4735.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  4736.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  4737.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  4738.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  4739.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  4740.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  4741.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  4742.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  4743.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  4744.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  4745.     - locked <0x00000001301e4078> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  4746.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  4747.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  4748.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  4749.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  4750.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  4751.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  4752.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  4753.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  4754.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  4755.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  4756.  
  4757. "RpcServer.default.FPBQ.Fifo.handler=7,queue=1,port=16020" #45 daemon prio=5 os_prio=0 tid=0x00007f9e373de000 nid=0x2341 waiting for monitor entry [0x00007f9debc8f000]
  4758.    java.lang.Thread.State: BLOCKED (on object monitor)
  4759.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  4760.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  4761.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  4762.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  4763.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  4764.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  4765.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  4766.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  4767.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  4768.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  4769.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  4770.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  4771.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  4772.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  4773.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  4774.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  4775.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  4776.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  4777.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  4778.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  4779.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  4780.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  4781.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  4782.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  4783.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  4784.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  4785.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  4786.     - locked <0x0000000130711428> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  4787.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  4788.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  4789.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  4790.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  4791.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  4792.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  4793.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  4794.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  4795.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  4796.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  4797.  
  4798. "RpcServer.default.FPBQ.Fifo.handler=6,queue=0,port=16020" #44 daemon prio=5 os_prio=0 tid=0x00007f9e373dc000 nid=0x2340 waiting for monitor entry [0x00007f9debd90000]
  4799.    java.lang.Thread.State: BLOCKED (on object monitor)
  4800.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  4801.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  4802.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  4803.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  4804.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  4805.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  4806.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  4807.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  4808.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  4809.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  4810.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  4811.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  4812.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  4813.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  4814.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  4815.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  4816.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  4817.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  4818.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  4819.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  4820.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  4821.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  4822.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  4823.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  4824.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  4825.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  4826.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  4827.     - locked <0x0000000130561958> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  4828.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  4829.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  4830.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  4831.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  4832.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  4833.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  4834.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  4835.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  4836.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  4837.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  4838.  
  4839. "RpcServer.default.FPBQ.Fifo.handler=5,queue=5,port=16020" #43 daemon prio=5 os_prio=0 tid=0x00007f9e373da000 nid=0x233f waiting for monitor entry [0x00007f9debe91000]
  4840.    java.lang.Thread.State: BLOCKED (on object monitor)
  4841.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  4842.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  4843.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  4844.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  4845.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  4846.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  4847.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  4848.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  4849.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  4850.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  4851.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  4852.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  4853.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  4854.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  4855.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  4856.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  4857.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  4858.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  4859.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  4860.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  4861.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  4862.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  4863.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  4864.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  4865.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  4866.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  4867.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  4868.     - locked <0x0000000121a35eb0> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  4869.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  4870.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  4871.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  4872.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  4873.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  4874.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  4875.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  4876.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  4877.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  4878.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  4879.  
  4880. "RpcServer.default.FPBQ.Fifo.handler=4,queue=4,port=16020" #42 daemon prio=5 os_prio=0 tid=0x00007f9e373d8800 nid=0x233e in Object.wait() [0x00007f9debf93000]
  4881.    java.lang.Thread.State: WAITING (on object monitor)
  4882.     at java.lang.Object.wait(Native Method)
  4883.     at java.lang.Object.wait(Object.java:502)
  4884.     at org.apache.hadoop.hbase.util.IdLock.getLockEntry(IdLock.java:81)
  4885.     - locked <0x0000000130a70b98> (a org.apache.hadoop.hbase.util.IdLock$Entry)
  4886.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:416)
  4887.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  4888.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  4889.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  4890.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  4891.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  4892.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  4893.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  4894.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  4895.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  4896.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  4897.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  4898.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  4899.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  4900.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  4901.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  4902.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  4903.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  4904.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  4905.     - locked <0x0000000130a827a0> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  4906.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  4907.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  4908.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  4909.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  4910.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  4911.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  4912.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  4913.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  4914.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  4915.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  4916.  
  4917. "RpcServer.default.FPBQ.Fifo.handler=3,queue=3,port=16020" #41 daemon prio=5 os_prio=0 tid=0x00007f9e373d6800 nid=0x233d runnable [0x00007f9dec093000]
  4918.    java.lang.Thread.State: RUNNABLE
  4919.     at java.net.PlainSocketImpl.socketConnect(Native Method)
  4920.     at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
  4921.     - locked <0x000000012f4ab798> (a java.net.SocksSocketImpl)
  4922.     at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
  4923.     at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
  4924.     at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
  4925.     at java.net.Socket.connect(Socket.java:589)
  4926.     at com.amazon.ws.emr.hadoop.fs.shaded.org.apache.http.conn.ssl.SSLConnectionSocketFactory.connectSocket(SSLConnectionSocketFactory.java:337)
  4927.     at com.amazon.ws.emr.hadoop.fs.shaded.com.amazonaws.http.conn.ssl.SdkTLSSocketFactory.connectSocket(SdkTLSSocketFactory.java:142)
  4928.     at com.amazon.ws.emr.hadoop.fs.shaded.org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:141)
  4929.     at com.amazon.ws.emr.hadoop.fs.shaded.org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:353)
  4930.     at sun.reflect.GeneratedMethodAccessor51.invoke(Unknown Source)
  4931.     at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
  4932.     at java.lang.reflect.Method.invoke(Method.java:498)
  4933.     at com.amazon.ws.emr.hadoop.fs.shaded.com.amazonaws.http.conn.ClientConnectionManagerFactory$Handler.invoke(ClientConnectionManagerFactory.java:76)
  4934.     at com.amazon.ws.emr.hadoop.fs.shaded.com.amazonaws.http.conn.$Proxy28.connect(Unknown Source)
  4935.     at com.amazon.ws.emr.hadoop.fs.shaded.org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:380)
  4936.     at com.amazon.ws.emr.hadoop.fs.shaded.org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:236)
  4937.     at com.amazon.ws.emr.hadoop.fs.shaded.org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
  4938.     at com.amazon.ws.emr.hadoop.fs.shaded.org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
  4939.     at com.amazon.ws.emr.hadoop.fs.shaded.org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
  4940.     at com.amazon.ws.emr.hadoop.fs.shaded.org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
  4941.     at com.amazon.ws.emr.hadoop.fs.shaded.com.amazonaws.http.apache.client.impl.SdkHttpClient.execute(SdkHttpClient.java:72)
  4942.     at com.amazon.ws.emr.hadoop.fs.shaded.com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1256)
  4943.     at com.amazon.ws.emr.hadoop.fs.shaded.com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1072)
  4944.     at com.amazon.ws.emr.hadoop.fs.shaded.com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:745)
  4945.     at com.amazon.ws.emr.hadoop.fs.shaded.com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:719)
  4946.     at com.amazon.ws.emr.hadoop.fs.shaded.com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:701)
  4947.     at com.amazon.ws.emr.hadoop.fs.shaded.com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:669)
  4948.     at com.amazon.ws.emr.hadoop.fs.shaded.com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:651)
  4949.     at com.amazon.ws.emr.hadoop.fs.shaded.com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:515)
  4950.     at com.amazon.ws.emr.hadoop.fs.shaded.com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:4443)
  4951.     at com.amazon.ws.emr.hadoop.fs.shaded.com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:4390)
  4952.     at com.amazon.ws.emr.hadoop.fs.shaded.com.amazonaws.services.s3.AmazonS3Client.getObject(AmazonS3Client.java:1427)
  4953.     at com.amazon.ws.emr.hadoop.fs.s3.lite.call.GetObjectCall.perform(GetObjectCall.java:21)
  4954.     at com.amazon.ws.emr.hadoop.fs.s3.lite.call.GetObjectCall.perform(GetObjectCall.java:8)
  4955.     at com.amazon.ws.emr.hadoop.fs.s3.lite.executor.GlobalS3Executor.execute(GlobalS3Executor.java:91)
  4956.     at com.amazon.ws.emr.hadoop.fs.s3.lite.AmazonS3LiteClient.invoke(AmazonS3LiteClient.java:184)
  4957.     at com.amazon.ws.emr.hadoop.fs.s3.lite.AmazonS3LiteClient.getObject(AmazonS3LiteClient.java:101)
  4958.     at com.amazon.ws.emr.hadoop.fs.s3.GetObjectInputStreamWithInfoFactory.create(GetObjectInputStreamWithInfoFactory.java:40)
  4959.     at com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream.open(S3FSInputStream.java:68)
  4960.     at com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream.retrieveInputStreamWithInfo(S3FSInputStream.java:192)
  4961.     at com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream.reopenStream(S3FSInputStream.java:180)
  4962.     - eliminated <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  4963.     at com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream.seekStream(S3FSInputStream.java:246)
  4964.     - eliminated <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  4965.     at com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream.read(S3FSInputStream.java:117)
  4966.     - locked <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  4967.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:75)
  4968.     - locked <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  4969.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  4970.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  4971.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  4972.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  4973.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  4974.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  4975.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  4976.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  4977.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  4978.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  4979.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  4980.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  4981.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  4982.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  4983.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  4984.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  4985.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  4986.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  4987.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  4988.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  4989.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  4990.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  4991.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  4992.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  4993.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  4994.     - locked <0x0000000125971b78> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  4995.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  4996.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  4997.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  4998.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  4999.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  5000.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  5001.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  5002.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  5003.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  5004.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  5005.  
  5006. "RpcServer.default.FPBQ.Fifo.handler=2,queue=2,port=16020" #40 daemon prio=5 os_prio=0 tid=0x00007f9e373d4800 nid=0x233c waiting for monitor entry [0x00007f9dec194000]
  5007.    java.lang.Thread.State: BLOCKED (on object monitor)
  5008.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  5009.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  5010.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  5011.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  5012.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  5013.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  5014.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  5015.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  5016.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  5017.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  5018.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  5019.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  5020.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  5021.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  5022.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  5023.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  5024.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  5025.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  5026.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  5027.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  5028.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  5029.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  5030.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  5031.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  5032.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  5033.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  5034.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  5035.     - locked <0x00000001301e9508> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  5036.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  5037.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  5038.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  5039.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  5040.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  5041.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  5042.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  5043.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  5044.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  5045.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  5046.  
  5047. "RpcServer.default.FPBQ.Fifo.handler=1,queue=1,port=16020" #39 daemon prio=5 os_prio=0 tid=0x00007f9e373d3000 nid=0x233b waiting for monitor entry [0x00007f9dec295000]
  5048.    java.lang.Thread.State: BLOCKED (on object monitor)
  5049.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  5050.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  5051.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  5052.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  5053.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  5054.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  5055.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  5056.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  5057.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  5058.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  5059.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  5060.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  5061.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  5062.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  5063.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  5064.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  5065.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  5066.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  5067.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  5068.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  5069.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  5070.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  5071.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  5072.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  5073.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  5074.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  5075.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  5076.     - locked <0x000000012e5d7bf0> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  5077.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  5078.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  5079.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  5080.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  5081.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  5082.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  5083.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  5084.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  5085.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  5086.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  5087.  
  5088. "RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=16020" #38 daemon prio=5 os_prio=0 tid=0x00007f9e373d1000 nid=0x233a waiting for monitor entry [0x00007f9dec396000]
  5089.    java.lang.Thread.State: BLOCKED (on object monitor)
  5090.     at org.apache.hadoop.fs.FSInputStream.read(FSInputStream.java:70)
  5091.     - waiting to lock <0x00000001dedb3c78> (a com.amazon.ws.emr.hadoop.fs.s3.S3FSInputStream)
  5092.     at org.apache.hadoop.fs.BufferedFSInputStream.read(BufferedFSInputStream.java:108)
  5093.     at org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:92)
  5094.     at org.apache.hadoop.hbase.io.hfile.HFileBlock.positionalReadWithExtra(HFileBlock.java:781)
  5095.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1502)
  5096.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1770)
  5097.     at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1596)
  5098.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:454)
  5099.     at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269)
  5100.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:651)
  5101.     at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:631)
  5102.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:322)
  5103.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:229)
  5104.     at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:445)
  5105.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:361)
  5106.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:307)
  5107.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:269)
  5108.     at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:1028)
  5109.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:1017)
  5110.     at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(StoreScanner.java:743)
  5111.     at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:626)
  5112.     at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:152)
  5113.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:6290)
  5114.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:6450)
  5115.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:6222)
  5116.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6199)
  5117.     - locked <0x00000001308d5408> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl)
  5118.     at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:6186)
  5119.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7427)
  5120.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7385)
  5121.     at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:7378)
  5122.     at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2263)
  5123.     at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:36609)
  5124.     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2380)
  5125.     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
  5126.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:297)
  5127.     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:277)
  5128.  
  5129. "RpcServer.listener,port=16020" #15 daemon prio=5 os_prio=0 tid=0x00007f9e373cd000 nid=0x2339 runnable [0x00007f9dec498000]
  5130.    java.lang.Thread.State: RUNNABLE
  5131.     at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  5132.     at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
  5133.     at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
  5134.     at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
  5135.     - locked <0x0000000149ac2a50> (a sun.nio.ch.Util$3)
  5136.     - locked <0x0000000149ac2a38> (a java.util.Collections$UnmodifiableSet)
  5137.     - locked <0x0000000149ac0be0> (a sun.nio.ch.EPollSelectorImpl)
  5138.     at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
  5139.     at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
  5140.     at org.apache.hadoop.hbase.ipc.RpcServer$Listener.run(RpcServer.java:832)
  5141.  
  5142. "RpcServer.responder" #28 daemon prio=5 os_prio=0 tid=0x00007f9e373ca800 nid=0x2338 runnable [0x00007f9dec599000]
  5143.    java.lang.Thread.State: RUNNABLE
  5144.     at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  5145.     at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
  5146.     at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
  5147.     at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
  5148.     - locked <0x0000000149d25110> (a sun.nio.ch.Util$3)
  5149.     - locked <0x0000000149d250f8> (a java.util.Collections$UnmodifiableSet)
  5150.     - locked <0x0000000149d1f860> (a sun.nio.ch.EPollSelectorImpl)
  5151.     at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
  5152.     at org.apache.hadoop.hbase.ipc.RpcServer$Responder.doRunLoop(RpcServer.java:1072)
  5153.     at org.apache.hadoop.hbase.ipc.RpcServer$Responder.run(RpcServer.java:1019)
  5154.  
  5155. "main-EventThread" #37 daemon prio=5 os_prio=0 tid=0x00007f9e373b3000 nid=0x2337 waiting on condition [0x00007f9dec69a000]
  5156.    java.lang.Thread.State: WAITING (parking)
  5157.     at sun.misc.Unsafe.park(Native Method)
  5158.     - parking to wait for  <0x00000001512bf438> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  5159.     at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  5160.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  5161.     at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  5162.     at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:504)
  5163.  
  5164. "main-SendThread(ip-172-20-113-124.us-west-2.compute.internal:2181)" #36 daemon prio=5 os_prio=0 tid=0x00007f9e37394000 nid=0x2336 runnable [0x00007f9dec79b000]
  5165.    java.lang.Thread.State: RUNNABLE
  5166.     at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  5167.     at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
  5168.     at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
  5169.     at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
  5170.     - locked <0x00000001511e5fd8> (a sun.nio.ch.Util$3)
  5171.     - locked <0x00000001511e6080> (a java.util.Collections$UnmodifiableSet)
  5172.     - locked <0x000000014a0eb1c8> (a sun.nio.ch.EPollSelectorImpl)
  5173.     at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
  5174.     at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)
  5175.     at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1145)
  5176.  
  5177. "client DomainSocketWatcher" #35 daemon prio=5 os_prio=0 tid=0x00007f9e36bbc800 nid=0x2334 runnable [0x00007f9deca9c000]
  5178.    java.lang.Thread.State: RUNNABLE
  5179.     at org.apache.hadoop.net.unix.DomainSocketWatcher.doPoll0(Native Method)
  5180.     at org.apache.hadoop.net.unix.DomainSocketWatcher.access$900(DomainSocketWatcher.java:52)
  5181.     at org.apache.hadoop.net.unix.DomainSocketWatcher$2.run(DomainSocketWatcher.java:503)
  5182.     at java.lang.Thread.run(Thread.java:748)
  5183.  
  5184. "org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner" #33 daemon prio=5 os_prio=0 tid=0x00007f9e3701e800 nid=0x232f in Object.wait() [0x00007f9ded19d000]
  5185.    java.lang.Thread.State: WAITING (on object monitor)
  5186.     at java.lang.Object.wait(Native Method)
  5187.     at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:144)
  5188.     - locked <0x00000001511e4ef8> (a java.lang.ref.ReferenceQueue$Lock)
  5189.     at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:165)
  5190.     at org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:3260)
  5191.     at java.lang.Thread.run(Thread.java:748)
  5192.  
  5193. "java-sdk-http-connection-reaper" #31 daemon prio=5 os_prio=0 tid=0x00007f9e36e33000 nid=0x232e waiting on condition [0x00007f9def083000]
  5194.    java.lang.Thread.State: TIMED_WAITING (sleeping)
  5195.     at java.lang.Thread.sleep(Native Method)
  5196.     at com.amazon.ws.emr.hadoop.fs.shaded.com.amazonaws.http.IdleConnectionReaper.run(IdleConnectionReaper.java:181)
  5197.  
  5198. "com.amazon.ws.emr.hadoop.fs.shaded.com.google.inject.internal.util.$Finalizer" #29 daemon prio=5 os_prio=0 tid=0x00007f9e3699f800 nid=0x2329 in Object.wait() [0x00007f9def6ac000]
  5199.    java.lang.Thread.State: WAITING (on object monitor)
  5200.     at java.lang.Object.wait(Native Method)
  5201.     - waiting on <0x0000000151184920> (a java.lang.ref.ReferenceQueue$Lock)
  5202.     at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:144)
  5203.     - locked <0x0000000151184920> (a java.lang.ref.ReferenceQueue$Lock)
  5204.     at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:165)
  5205.     at com.amazon.ws.emr.hadoop.fs.shaded.com.google.inject.internal.util.$Finalizer.run(Finalizer.java:114)
  5206.  
  5207. "HBase-Metrics2-1" #27 daemon prio=5 os_prio=0 tid=0x00007f9e36775800 nid=0x2325 waiting on condition [0x00007f9defdaf000]
  5208.    java.lang.Thread.State: TIMED_WAITING (parking)
  5209.     at sun.misc.Unsafe.park(Native Method)
  5210.     - parking to wait for  <0x00000001499b9388> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  5211.     at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
  5212.     at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
  5213.     at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
  5214.     at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
  5215.     at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
  5216.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  5217.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  5218.     at java.lang.Thread.run(Thread.java:748)
  5219.  
  5220. "RpcServer.reader=9,bindAddress=ip-172-20-113-118.us-west-2.compute.internal,port=16020" #25 daemon prio=5 os_prio=0 tid=0x00007f9e366b1800 nid=0x2323 runnable [0x00007f9df01b1000]
  5221.    java.lang.Thread.State: RUNNABLE
  5222.     at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  5223.     at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
  5224.     at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
  5225.     at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
  5226.     - locked <0x00000001499b1ea8> (a sun.nio.ch.Util$3)
  5227.     - locked <0x00000001499b1ec0> (a java.util.Collections$UnmodifiableSet)
  5228.     - locked <0x00000001499b9538> (a sun.nio.ch.EPollSelectorImpl)
  5229.     at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
  5230.     at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
  5231.     at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:714)
  5232.     - locked <0x00000001499b1ed8> (a org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader)
  5233.     at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:701)
  5234.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  5235.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  5236.     at java.lang.Thread.run(Thread.java:748)
  5237.  
  5238. "RpcServer.reader=8,bindAddress=ip-172-20-113-118.us-west-2.compute.internal,port=16020" #24 daemon prio=5 os_prio=0 tid=0x00007f9e36696800 nid=0x2322 runnable [0x00007f9df02b2000]
  5239.    java.lang.Thread.State: RUNNABLE
  5240.     at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  5241.     at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
  5242.     at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
  5243.     at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
  5244.     - locked <0x00000001499bbda0> (a sun.nio.ch.Util$3)
  5245.     - locked <0x00000001499bbdb8> (a java.util.Collections$UnmodifiableSet)
  5246.     - locked <0x00000001499b99b8> (a sun.nio.ch.EPollSelectorImpl)
  5247.     at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
  5248.     at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
  5249.     at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:714)
  5250.     - locked <0x00000001499bbdd0> (a org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader)
  5251.     at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:701)
  5252.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  5253.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  5254.     at java.lang.Thread.run(Thread.java:748)
  5255.  
  5256. "RpcServer.reader=7,bindAddress=ip-172-20-113-118.us-west-2.compute.internal,port=16020" #23 daemon prio=5 os_prio=0 tid=0x00007f9e3667b800 nid=0x2321 runnable [0x00007f9df03b3000]
  5257.    java.lang.Thread.State: RUNNABLE
  5258.     at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  5259.     at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
  5260.     at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
  5261.     at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
  5262.     - locked <0x00000001499bcaa0> (a sun.nio.ch.Util$3)
  5263.     - locked <0x00000001499bcab8> (a java.util.Collections$UnmodifiableSet)
  5264.     - locked <0x00000001499bc168> (a sun.nio.ch.EPollSelectorImpl)
  5265.     at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
  5266.     at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
  5267.     at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:714)
  5268.     - locked <0x00000001499bcad0> (a org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader)
  5269.     at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:701)
  5270.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  5271.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  5272.     at java.lang.Thread.run(Thread.java:748)
  5273.  
  5274. "RpcServer.reader=6,bindAddress=ip-172-20-113-118.us-west-2.compute.internal,port=16020" #22 daemon prio=5 os_prio=0 tid=0x00007f9e36660800 nid=0x2320 runnable [0x00007f9df04b4000]
  5275.    java.lang.Thread.State: RUNNABLE
  5276.     at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  5277.     at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
  5278.     at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
  5279.     at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
  5280.     - locked <0x00000001499b93b8> (a sun.nio.ch.Util$3)
  5281.     - locked <0x00000001499b93d0> (a java.util.Collections$UnmodifiableSet)
  5282.     - locked <0x00000001499bcf68> (a sun.nio.ch.EPollSelectorImpl)
  5283.     at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
  5284.     at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
  5285.     at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:714)
  5286.     - locked <0x00000001499b93e8> (a org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader)
  5287.     at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:701)
  5288.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  5289.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  5290.     at java.lang.Thread.run(Thread.java:748)
  5291.  
  5292. "RpcServer.reader=5,bindAddress=ip-172-20-113-118.us-west-2.compute.internal,port=16020" #21 daemon prio=5 os_prio=0 tid=0x00007f9e36646000 nid=0x231f runnable [0x00007f9df05b5000]
  5293.    java.lang.Thread.State: RUNNABLE
  5294.     at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  5295.     at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
  5296.     at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
  5297.     at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
  5298.     - locked <0x00000001499bf7d0> (a sun.nio.ch.Util$3)
  5299.     - locked <0x00000001499bf7e8> (a java.util.Collections$UnmodifiableSet)
  5300.     - locked <0x00000001499bf398> (a sun.nio.ch.EPollSelectorImpl)
  5301.     at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
  5302.     at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
  5303.     at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:714)
  5304.     - locked <0x00000001499bf800> (a org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader)
  5305.     at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:701)
  5306.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  5307.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  5308.     at java.lang.Thread.run(Thread.java:748)
  5309.  
  5310. "RpcServer.reader=4,bindAddress=ip-172-20-113-118.us-west-2.compute.internal,port=16020" #20 daemon prio=5 os_prio=0 tid=0x00007f9e3662b000 nid=0x231e runnable [0x00007f9df06b6000]
  5311.    java.lang.Thread.State: RUNNABLE
  5312.     at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  5313.     at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
  5314.     at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
  5315.     at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
  5316.     - locked <0x00000001499c1d80> (a sun.nio.ch.Util$3)
  5317.     - locked <0x00000001499c1d98> (a java.util.Collections$UnmodifiableSet)
  5318.     - locked <0x00000001499c1948> (a sun.nio.ch.EPollSelectorImpl)
  5319.     at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
  5320.     at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
  5321.     at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:714)
  5322.     - locked <0x00000001499c1db0> (a org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader)
  5323.     at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:701)
  5324.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  5325.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  5326.     at java.lang.Thread.run(Thread.java:748)
  5327.  
  5328. "RpcServer.reader=3,bindAddress=ip-172-20-113-118.us-west-2.compute.internal,port=16020" #19 daemon prio=5 os_prio=0 tid=0x00007f9e36610000 nid=0x231d runnable [0x00007f9df07b7000]
  5329.    java.lang.Thread.State: RUNNABLE
  5330.     at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  5331.     at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
  5332.     at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
  5333.     at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
  5334.     - locked <0x00000001499b9400> (a sun.nio.ch.Util$3)
  5335.     - locked <0x00000001499b9418> (a java.util.Collections$UnmodifiableSet)
  5336.     - locked <0x00000001499bcff8> (a sun.nio.ch.EPollSelectorImpl)
  5337.     at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
  5338.     at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
  5339.     at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:714)
  5340.     - locked <0x00000001499b9430> (a org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader)
  5341.     at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:701)
  5342.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  5343.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  5344.     at java.lang.Thread.run(Thread.java:748)
  5345.  
  5346. "RpcServer.reader=2,bindAddress=ip-172-20-113-118.us-west-2.compute.internal,port=16020" #18 daemon prio=5 os_prio=0 tid=0x00007f9e365f5800 nid=0x231c runnable [0x00007f9df08b8000]
  5347.    java.lang.Thread.State: RUNNABLE
  5348.     at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  5349.     at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
  5350.     at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
  5351.     at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
  5352.     - locked <0x00000001499c1dc8> (a sun.nio.ch.Util$3)
  5353.     - locked <0x00000001499c1de0> (a java.util.Collections$UnmodifiableSet)
  5354.     - locked <0x00000001499c19d8> (a sun.nio.ch.EPollSelectorImpl)
  5355.     at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
  5356.     at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
  5357.     at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:714)
  5358.     - locked <0x00000001499c1df8> (a org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader)
  5359.     at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:701)
  5360.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  5361.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  5362.     at java.lang.Thread.run(Thread.java:748)
  5363.  
  5364. "RpcServer.reader=1,bindAddress=ip-172-20-113-118.us-west-2.compute.internal,port=16020" #17 daemon prio=5 os_prio=0 tid=0x00007f9e36582000 nid=0x231b runnable [0x00007f9df09b9000]
  5365.    java.lang.Thread.State: RUNNABLE
  5366.     at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  5367.     at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
  5368.     at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
  5369.     at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
  5370.     - locked <0x00000001499b9448> (a sun.nio.ch.Util$3)
  5371.     - locked <0x00000001499b9460> (a java.util.Collections$UnmodifiableSet)
  5372.     - locked <0x00000001499bd088> (a sun.nio.ch.EPollSelectorImpl)
  5373.     at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
  5374.     at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
  5375.     at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:714)
  5376.     - locked <0x00000001499b9478> (a org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader)
  5377.     at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:701)
  5378.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  5379.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  5380.     at java.lang.Thread.run(Thread.java:748)
  5381.  
  5382. "RpcServer.reader=0,bindAddress=ip-172-20-113-118.us-west-2.compute.internal,port=16020" #16 daemon prio=5 os_prio=0 tid=0x00007f9e36581000 nid=0x231a runnable [0x00007f9df1cbe000]
  5383.    java.lang.Thread.State: RUNNABLE
  5384.     at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  5385.     at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
  5386.     at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
  5387.     at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
  5388.     - locked <0x00000001499bcae8> (a sun.nio.ch.Util$3)
  5389.     - locked <0x00000001499bcb00> (a java.util.Collections$UnmodifiableSet)
  5390.     - locked <0x00000001499bc1f8> (a sun.nio.ch.EPollSelectorImpl)
  5391.     at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
  5392.     at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
  5393.     at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:714)
  5394.     - locked <0x00000001499bcb18> (a org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader)
  5395.     at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:701)
  5396.     at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  5397.     at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  5398.     at java.lang.Thread.run(Thread.java:748)
  5399.  
  5400. "Service Thread" #10 daemon prio=9 os_prio=0 tid=0x00007f9e341d7800 nid=0x2311 runnable [0x0000000000000000]
  5401.    java.lang.Thread.State: RUNNABLE
  5402.  
  5403. "C1 CompilerThread3" #9 daemon prio=9 os_prio=0 tid=0x00007f9e341ab800 nid=0x2310 waiting on condition [0x0000000000000000]
  5404.    java.lang.Thread.State: RUNNABLE
  5405.  
  5406. "C2 CompilerThread2" #8 daemon prio=9 os_prio=0 tid=0x00007f9e341a9000 nid=0x230f waiting on condition [0x0000000000000000]
  5407.    java.lang.Thread.State: RUNNABLE
  5408.  
  5409. "C2 CompilerThread1" #7 daemon prio=9 os_prio=0 tid=0x00007f9e341a7000 nid=0x230e waiting on condition [0x0000000000000000]
  5410.    java.lang.Thread.State: RUNNABLE
  5411.  
  5412. "C2 CompilerThread0" #6 daemon prio=9 os_prio=0 tid=0x00007f9e341a4800 nid=0x230d waiting on condition [0x0000000000000000]
  5413.    java.lang.Thread.State: RUNNABLE
  5414.  
  5415. "Signal Dispatcher" #5 daemon prio=9 os_prio=0 tid=0x00007f9e34196000 nid=0x230c runnable [0x0000000000000000]
  5416.    java.lang.Thread.State: RUNNABLE
  5417.  
  5418. "Surrogate Locker Thread (Concurrent GC)" #4 daemon prio=9 os_prio=0 tid=0x00007f9e34194000 nid=0x230b waiting on condition [0x0000000000000000]
  5419.    java.lang.Thread.State: RUNNABLE
  5420.  
  5421. "Finalizer" #3 daemon prio=8 os_prio=0 tid=0x00007f9e3416a000 nid=0x230a in Object.wait() [0x00007f9e24c2c000]
  5422.    java.lang.Thread.State: WAITING (on object monitor)
  5423.     at java.lang.Object.wait(Native Method)
  5424.     at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:144)
  5425.     - locked <0x00000001499b9490> (a java.lang.ref.ReferenceQueue$Lock)
  5426.     at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:165)
  5427.     at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:216)
  5428.  
  5429. "Reference Handler" #2 daemon prio=10 os_prio=0 tid=0x00007f9e34165800 nid=0x2309 in Object.wait() [0x00007f9e24d2d000]
  5430.    java.lang.Thread.State: WAITING (on object monitor)
  5431.     at java.lang.Object.wait(Native Method)
  5432.     at java.lang.Object.wait(Object.java:502)
  5433.     at java.lang.ref.Reference.tryHandlePending(Reference.java:191)
  5434.     - locked <0x00000001499bcb30> (a java.lang.ref.Reference$Lock)
  5435.     at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153)
  5436.  
  5437. "main" #1 prio=5 os_prio=0 tid=0x00007f9e3405d800 nid=0x22fc in Object.wait() [0x00007f9e3e2d2000]
  5438.    java.lang.Thread.State: WAITING (on object monitor)
  5439.     at java.lang.Object.wait(Native Method)
  5440.     - waiting on <0x00000001499c05e8> (a java.lang.Thread)
  5441.     at java.lang.Thread.join(Thread.java:1252)
  5442.     - locked <0x00000001499c05e8> (a java.lang.Thread)
  5443.     at java.lang.Thread.join(Thread.java:1326)
  5444.     at org.apache.hadoop.hbase.util.HasThread.join(HasThread.java:90)
  5445.     at org.apache.hadoop.hbase.regionserver.HRegionServerCommandLine.start(HRegionServerCommandLine.java:66)
  5446.     at org.apache.hadoop.hbase.regionserver.HRegionServerCommandLine.run(HRegionServerCommandLine.java:87)
  5447.     at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76)
  5448.     at org.apache.hadoop.hbase.util.ServerCommandLine.doMain(ServerCommandLine.java:127)
  5449.     at org.apache.hadoop.hbase.regionserver.HRegionServer.main(HRegionServer.java:2831)
  5450.  
  5451. "VM Thread" os_prio=0 tid=0x00007f9e3415b800 nid=0x2308 runnable
  5452.  
  5453. "Gang worker#0 (Parallel GC Threads)" os_prio=0 tid=0x00007f9e3406c000 nid=0x22fd runnable
  5454.  
  5455. "Gang worker#1 (Parallel GC Threads)" os_prio=0 tid=0x00007f9e3406d800 nid=0x22fe runnable
  5456.  
  5457. "Gang worker#2 (Parallel GC Threads)" os_prio=0 tid=0x00007f9e3406f800 nid=0x22ff runnable
  5458.  
  5459. "Gang worker#3 (Parallel GC Threads)" os_prio=0 tid=0x00007f9e34071800 nid=0x2300 runnable
  5460.  
  5461. "Gang worker#4 (Parallel GC Threads)" os_prio=0 tid=0x00007f9e34073000 nid=0x2301 runnable
  5462.  
  5463. "Gang worker#5 (Parallel GC Threads)" os_prio=0 tid=0x00007f9e34075000 nid=0x2302 runnable
  5464.  
  5465. "Gang worker#6 (Parallel GC Threads)" os_prio=0 tid=0x00007f9e34077000 nid=0x2303 runnable
  5466.  
  5467. "Gang worker#7 (Parallel GC Threads)" os_prio=0 tid=0x00007f9e34078800 nid=0x2304 runnable
  5468.  
  5469. "Concurrent Mark-Sweep GC Thread" os_prio=0 tid=0x00007f9e340ea000 nid=0x2307 runnable
  5470.  
  5471. "Gang worker#0 (Parallel CMS Threads)" os_prio=0 tid=0x00007f9e340e6000 nid=0x2305 runnable
  5472.  
  5473. "Gang worker#1 (Parallel CMS Threads)" os_prio=0 tid=0x00007f9e340e7800 nid=0x2306 runnable
  5474.  
  5475. "VM Periodic Task Thread" os_prio=0 tid=0x00007f9e341da000 nid=0x2312 waiting on condition
  5476.  
  5477. JNI global references: 296
RAW Paste Data