-PROD- prodcon@wchddb37pxdu:/prod/hadoop/prod/hbase-0.92.1/logs$cat hbase-prodcon-regionserver-wchddb37pxdu.prod.oclc.org.out SLF4J: Class path contains multiple SLF4J bindings. SLF4J: Found binding in [jar:file:/drive1/hadoop/20120522/hbase-0.92.1/lib/slf4j-log4j12-1.5.8.jar!/org/slf4j/impl/StaticLoggerBinder.class] SLF4J: Found binding in [jar:file:/drive1/hadoop/20120522/hadoop-0.20.2-cdh3u3/lib/slf4j-log4j12-1.4.3.jar!/org/slf4j/impl/StaticLoggerBinder.class] SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation. 2012-08-06 11:46:46 Full thread dump Java HotSpot(TM) 64-Bit Server VM (20.0-b11 mixed mode): "IPC Client (47) connection to /10.192.98.52:37073 from blk_81630121065287353_2238161" daemon prio=10 tid=0x00002aaacc7b7000 nid=0x1c8f in Object.wait() [0x000000004e0ef000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:680) - locked <0x00000005840a63a0> (a org.apache.hadoop.ipc.Client$Connection) at org.apache.hadoop.ipc.Client$Connection.run(Client.java:723) "IPC Client (47) connection to wchddb01pxdu.prod.oclc.org/10.192.98.16:9008 from prodcon" daemon prio=10 tid=0x00002aaacc3a5000 nid=0x1c8e in Object.wait() [0x000000004e3f2000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:680) - locked <0x0000000583b67d70> (a org.apache.hadoop.ipc.Client$Connection) at org.apache.hadoop.ipc.Client$Connection.run(Client.java:723) "1772284907@qtp-1068922666-14" prio=10 tid=0x00002aaad8488000 nid=0x1b27 in Object.wait() [0x000000004e5f4000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626) - locked <0x00000005835d94a0> (a org.mortbay.thread.QueuedThreadPool$PoolThread) "RMI TCP Connection(idle)" daemon prio=10 tid=0x00002aaad8490000 nid=0x1985 waiting on condition [0x000000004f503000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000058b4d4548> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:424) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:323) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:874) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:945) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:662) "915001464@qtp-1068922666-13" prio=10 tid=0x00002aaad8287800 nid=0x1949 in Object.wait() [0x000000004e2f1000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626) - locked <0x000000058057a0f0> (a org.mortbay.thread.QueuedThreadPool$PoolThread) "ResponseProcessor for block blk_81630121065287353_2238161" daemon prio=10 tid=0x00002aaabfe1c800 nid=0x13bb runnable [0x000000004e1f0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) - locked <0x0000000587f20758> (a sun.nio.ch.Util$2) - locked <0x0000000587f20748> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000587f206b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:332) at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:155) at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:128) at java.io.DataInputStream.readFully(DataInputStream.java:178) at java.io.DataInputStream.readLong(DataInputStream.java:399) at org.apache.hadoop.hdfs.protocol.DataTransferProtocol$PipelineAck.readFields(DataTransferProtocol.java:120) at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream$ResponseProcessor.run(DFSClient.java:2956) "DataStreamer for file /hbase/.logs/wchddb37pxdu.prod.oclc.org,9009,1337721421002/wchddb37pxdu.prod.oclc.org%2C9009%2C1337721421002.1344267769705 block blk_81630121065287353_2238161" daemon prio=10 tid=0x00002aaad81ab000 nid=0x13b4 in Object.wait() [0x000000004f200000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream$DataStreamer.run(DFSClient.java:2805) - locked <0x00000006e8c6b168> (a java.util.LinkedList) "sendParams-166" daemon prio=10 tid=0x00002aaacca57000 nid=0x2b0d waiting on condition [0x0000000066358000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000058b300eb0> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:424) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:323) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:874) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:945) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:662) "ReplicationExecutor-0" prio=10 tid=0x00002aaaccbd6000 nid=0x5e94 waiting on condition [0x0000000058a80000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000058b89f790> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:662) "IPC Client (47) connection to wchddb02pxdu.prod.oclc.org/10.192.98.17:9010 from prodcon" daemon prio=10 tid=0x00002aaaccc76800 nid=0x59df in Object.wait() [0x000000007ff3f000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.ipc.HBaseClient$Connection.waitForWork(HBaseClient.java:459) - locked <0x000000058fbd02e0> (a org.apache.hadoop.hbase.ipc.HBaseClient$Connection) at org.apache.hadoop.hbase.ipc.HBaseClient$Connection.run(HBaseClient.java:504) "RS_CLOSE_REGION-wchddb37pxdu.prod.oclc.org,9009,1337721421002-2" prio=10 tid=0x00002aaabf26d800 nid=0x3dd2 waiting on condition [0x00000000783c4000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000058b393590> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:662) "RS_CLOSE_REGION-wchddb37pxdu.prod.oclc.org,9009,1337721421002-1" prio=10 tid=0x0000000052de3800 nid=0x3dd1 waiting on condition [0x00000000782c3000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000058b393590> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:662) "RS_CLOSE_REGION-wchddb37pxdu.prod.oclc.org,9009,1337721421002-0" prio=10 tid=0x0000000052e1e000 nid=0x3dcc waiting on condition [0x0000000077abb000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000058b393590> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:662) "regionserver9009-splits-1337730044584" daemon prio=10 tid=0x0000000052868000 nid=0x1e64 waiting on condition [0x0000000056f65000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000058b8a3088> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:662) "regionserver9009-largeCompactions-1337726358700" daemon prio=10 tid=0x00002aaac0b55800 nid=0x973 waiting on condition [0x00000000777b8000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000058b8a16d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987) at java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:220) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:662) "regionserver9009-EventThread.replicationSource,1" daemon prio=10 tid=0x00002aaacc980000 nid=0xcca waiting on condition [0x0000000043544000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.sleepForRetries(ReplicationSource.java:550) at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.run(ReplicationSource.java:362) "regionserver9009-EventThread" daemon prio=10 tid=0x0000000052a50800 nid=0xcc9 waiting on condition [0x000000004909f000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000058caaa060> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:493) "regionserver9009-SendThread(wchddb02pxwe.prod.oclc.org:9011)" daemon prio=10 tid=0x0000000052946800 nid=0xcc8 runnable [0x0000000040cf6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) - locked <0x000000058caaa000> (a sun.nio.ch.Util$2) - locked <0x000000058caaa018> (a java.util.Collections$UnmodifiableSet) - locked <0x000000058cab5cf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:274) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1035) "regionserver9009-smallCompactions-1337721650410" daemon prio=10 tid=0x0000000052866000 nid=0x130 waiting on condition [0x0000000049baa000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000058b8a1618> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987) at java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:220) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:662) "RS_OPEN_REGION-wchddb37pxdu.prod.oclc.org,9009,1337721421002-2" prio=10 tid=0x00000000528f8000 nid=0x7fdd waiting on condition [0x00000000499a8000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000058b3938d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:662) "RS_OPEN_REGION-wchddb37pxdu.prod.oclc.org,9009,1337721421002-1" prio=10 tid=0x0000000053148000 nid=0x7fdc waiting on condition [0x000000004a5b4000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000058b3938d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:662) "RS_OPEN_REGION-wchddb37pxdu.prod.oclc.org,9009,1337721421002-0" prio=10 tid=0x0000000052e10000 nid=0x7fdb waiting on condition [0x00000000496a5000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000058b3938d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:662) "RMI Scheduler(0)" daemon prio=10 tid=0x0000000053126800 nid=0x7e4a waiting on condition [0x0000000049ead000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000058b7f1150> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2025) at java.util.concurrent.DelayQueue.take(DelayQueue.java:164) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:609) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:602) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:662) "SplitLogWorker-wchddb37pxdu.prod.oclc.org,9009,1337721421002" prio=10 tid=0x00002aaac0a59800 nid=0x6d83 in Object.wait() [0x000000004808f000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:485) at org.apache.hadoop.hbase.regionserver.SplitLogWorker.taskLoop(SplitLogWorker.java:205) - locked <0x000000058c87ada0> (a java.lang.Object) at org.apache.hadoop.hbase.regionserver.SplitLogWorker.run(SplitLogWorker.java:165) at java.lang.Thread.run(Thread.java:662) "PRI IPC Server handler 9 on 9009" daemon prio=10 tid=0x00002aaac0a57800 nid=0x6d82 waiting on condition [0x0000000047f8e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000058b300df0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1299) "PRI IPC Server handler 8 on 9009" daemon prio=10 tid=0x00002aaac0a55800 nid=0x6d81 waiting on condition [0x0000000047e8d000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000058b300df0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1299) "PRI IPC Server handler 7 on 9009" daemon prio=10 tid=0x00002aaac0a53800 nid=0x6d80 waiting on condition [0x0000000047d8c000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000058b300df0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1299) "PRI IPC Server handler 6 on 9009" daemon prio=10 tid=0x00002aaac0730000 nid=0x6d7f waiting on condition [0x0000000047c8b000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000058b300df0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1299) "PRI IPC Server handler 5 on 9009" daemon prio=10 tid=0x00002aaac072e000 nid=0x6d7e waiting on condition [0x0000000047b8a000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000058b300df0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1299) "PRI IPC Server handler 4 on 9009" daemon prio=10 tid=0x00002aaac072c000 nid=0x6d7d waiting on condition [0x0000000047a89000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000058b300df0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1299) "PRI IPC Server handler 3 on 9009" daemon prio=10 tid=0x00002aaac072a000 nid=0x6d7c waiting on condition [0x0000000047988000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000058b300df0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1299) "PRI IPC Server handler 2 on 9009" daemon prio=10 tid=0x00002aaac0728000 nid=0x6d7b waiting on condition [0x0000000047887000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000058b300df0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1299) "PRI IPC Server handler 1 on 9009" daemon prio=10 tid=0x00002aaac0726000 nid=0x6d7a waiting on condition [0x0000000047786000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000058b300df0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1299) "PRI IPC Server handler 0 on 9009" daemon prio=10 tid=0x00002aaac0724000 nid=0x6d79 waiting on condition [0x0000000047685000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000058b300df0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1299) "IPC Server handler 49 on 9009" daemon prio=10 tid=0x00002aaac0722800 nid=0x6d78 in Object.wait() [0x0000000047584000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:2051) - locked <0x00000006a84b9218> (a org.apache.hadoop.hbase.regionserver.HRegion) at org.apache.hadoop.hbase.regionserver.HRegion.delete(HRegion.java:1469) at org.apache.hadoop.hbase.regionserver.HRegionServer.delete(HRegionServer.java:2260) at sun.reflect.GeneratedMethodAccessor36.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1326) "IPC Server handler 48 on 9009" daemon prio=10 tid=0x00002aaac0047000 nid=0x6d77 in Object.wait() [0x0000000047483000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:2051) - locked <0x00000006a84b9218> (a org.apache.hadoop.hbase.regionserver.HRegion) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1717) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3076) at sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1326) "IPC Server handler 47 on 9009" daemon prio=10 tid=0x00002aaac0045000 nid=0x6d76 in Object.wait() [0x0000000047382000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:2051) - locked <0x00000006a84b9218> (a org.apache.hadoop.hbase.regionserver.HRegion) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1717) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3076) at sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1326) "IPC Server handler 46 on 9009" daemon prio=10 tid=0x00002aaac0043800 nid=0x6d75 in Object.wait() [0x0000000047281000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:2051) - locked <0x00000006a84b9218> (a org.apache.hadoop.hbase.regionserver.HRegion) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1717) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3076) at sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1326) "IPC Server handler 45 on 9009" daemon prio=10 tid=0x00002aaac0041800 nid=0x6d74 in Object.wait() [0x0000000047180000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:2051) - locked <0x00000006a84b9218> (a org.apache.hadoop.hbase.regionserver.HRegion) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1717) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3076) at sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1326) "IPC Server handler 44 on 9009" daemon prio=10 tid=0x00002aaac003f800 nid=0x6d73 in Object.wait() [0x000000004707f000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:2051) - locked <0x00000006a84b9218> (a org.apache.hadoop.hbase.regionserver.HRegion) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1717) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3076) at sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1326) "IPC Server handler 43 on 9009" daemon prio=10 tid=0x00002aaac003d800 nid=0x6d72 in Object.wait() [0x0000000046f7e000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:2051) - locked <0x00000006a84b9218> (a org.apache.hadoop.hbase.regionserver.HRegion) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1717) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3076) at sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1326) "IPC Server handler 42 on 9009" daemon prio=10 tid=0x00002aaac003b800 nid=0x6d71 in Object.wait() [0x0000000046e7d000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:2051) - locked <0x00000006a84b9218> (a org.apache.hadoop.hbase.regionserver.HRegion) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1717) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3076) at sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1326) "IPC Server handler 41 on 9009" daemon prio=10 tid=0x00002aaac0039800 nid=0x6d70 in Object.wait() [0x0000000046d7c000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:2051) - locked <0x00000006a84b9218> (a org.apache.hadoop.hbase.regionserver.HRegion) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1717) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3076) at sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1326) "IPC Server handler 40 on 9009" daemon prio=10 tid=0x00002aaac0a00800 nid=0x6d6f in Object.wait() [0x0000000046c7b000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:2051) - locked <0x00000006a84b9218> (a org.apache.hadoop.hbase.regionserver.HRegion) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1717) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3076) at sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1326) "IPC Server handler 39 on 9009" daemon prio=10 tid=0x00002aaac09fe800 nid=0x6d6e in Object.wait() [0x0000000046b7a000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:2051) - locked <0x00000006a84b9218> (a org.apache.hadoop.hbase.regionserver.HRegion) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1717) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3076) at sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1326) "IPC Server handler 38 on 9009" daemon prio=10 tid=0x00002aaac09fc800 nid=0x6d6d in Object.wait() [0x0000000046a79000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:2051) - locked <0x00000006a84b9218> (a org.apache.hadoop.hbase.regionserver.HRegion) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1717) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3076) at sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1326) "IPC Server handler 37 on 9009" daemon prio=10 tid=0x00002aaac09fa800 nid=0x6d6c in Object.wait() [0x0000000046978000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:2051) - locked <0x00000006a84b9218> (a org.apache.hadoop.hbase.regionserver.HRegion) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1717) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3076) at sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1326) "IPC Server handler 36 on 9009" daemon prio=10 tid=0x00002aaac09f8800 nid=0x6d6b in Object.wait() [0x0000000046877000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:2051) - locked <0x00000006a84b9218> (a org.apache.hadoop.hbase.regionserver.HRegion) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1717) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3076) at sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1326) "IPC Server handler 35 on 9009" daemon prio=10 tid=0x00002aaac09f6800 nid=0x6d6a in Object.wait() [0x0000000046776000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:2051) - locked <0x00000006a84b9218> (a org.apache.hadoop.hbase.regionserver.HRegion) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1717) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3076) at sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1326) "IPC Server handler 34 on 9009" daemon prio=10 tid=0x00002aaac09f4800 nid=0x6d69 in Object.wait() [0x0000000046675000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:2051) - locked <0x00000006a84b9218> (a org.apache.hadoop.hbase.regionserver.HRegion) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1717) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3076) at sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1326) "IPC Server handler 33 on 9009" daemon prio=10 tid=0x00002aaac09f2800 nid=0x6d68 in Object.wait() [0x0000000046574000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:2051) - locked <0x00000006a84b9218> (a org.apache.hadoop.hbase.regionserver.HRegion) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1717) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3076) at sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1326) "IPC Server handler 32 on 9009" daemon prio=10 tid=0x00002aaac08a9800 nid=0x6d67 in Object.wait() [0x0000000046473000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:2051) - locked <0x00000006a84b9218> (a org.apache.hadoop.hbase.regionserver.HRegion) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1717) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3076) at sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1326) "IPC Server handler 31 on 9009" daemon prio=10 tid=0x00002aaac08a7800 nid=0x6d66 in Object.wait() [0x0000000046372000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:2051) - locked <0x00000006a84b9218> (a org.apache.hadoop.hbase.regionserver.HRegion) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1717) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3076) at sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1326) "IPC Server handler 30 on 9009" daemon prio=10 tid=0x00002aaac08a5800 nid=0x6d65 in Object.wait() [0x0000000046271000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:2051) - locked <0x00000006a84b9218> (a org.apache.hadoop.hbase.regionserver.HRegion) at org.apache.hadoop.hbase.regionserver.HRegion.delete(HRegion.java:1469) at org.apache.hadoop.hbase.regionserver.HRegionServer.delete(HRegionServer.java:2260) at sun.reflect.GeneratedMethodAccessor36.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1326) "IPC Server handler 29 on 9009" daemon prio=10 tid=0x00002aaac08a3800 nid=0x6d64 runnable [0x0000000046170000] java.lang.Thread.State: RUNNABLE at org.apache.hadoop.hbase.regionserver.MemStore$MemStoreScanner.getNext(MemStore.java:691) at org.apache.hadoop.hbase.regionserver.MemStore$MemStoreScanner.seekInSubLists(MemStore.java:726) - locked <0x000000058d939c38> (a org.apache.hadoop.hbase.regionserver.MemStore$MemStoreScanner) at org.apache.hadoop.hbase.regionserver.MemStore$MemStoreScanner.reseek(MemStore.java:765) - locked <0x000000058d939c38> (a org.apache.hadoop.hbase.regionserver.MemStore$MemStoreScanner) at org.apache.hadoop.hbase.regionserver.KeyValueHeap$SeekType$1.seek(KeyValueHeap.java:57) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:277) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:248) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:436) - locked <0x000000058fd24658> (a org.apache.hadoop.hbase.regionserver.StoreScanner) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:322) - locked <0x000000058fd24658> (a org.apache.hadoop.hbase.regionserver.StoreScanner) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:138) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:2945) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:2901) - locked <0x000000058d939c70> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:2918) - locked <0x000000058d939c70> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl) at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:3693) at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:3585) at org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:1785) at sun.reflect.GeneratedMethodAccessor41.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1326) "IPC Server handler 28 on 9009" daemon prio=10 tid=0x00002aaac08a1800 nid=0x6d63 in Object.wait() [0x000000004606f000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:2051) - locked <0x00000006a84b9218> (a org.apache.hadoop.hbase.regionserver.HRegion) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1717) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3076) at sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1326) "IPC Server handler 27 on 9009" daemon prio=10 tid=0x00002aaac08a0000 nid=0x6d62 in Object.wait() [0x0000000045f6e000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:2051) - locked <0x00000006a84b9218> (a org.apache.hadoop.hbase.regionserver.HRegion) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1717) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3076) at sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1326) "IPC Server handler 26 on 9009" daemon prio=10 tid=0x00002aaac089e000 nid=0x6d61 in Object.wait() [0x0000000045e6d000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:2051) - locked <0x00000006a84b9218> (a org.apache.hadoop.hbase.regionserver.HRegion) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1717) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3076) at sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1326) "IPC Server handler 25 on 9009" daemon prio=10 tid=0x00002aaac0a7f800 nid=0x6d60 in Object.wait() [0x0000000045d6c000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:2051) - locked <0x00000006a84b9218> (a org.apache.hadoop.hbase.regionserver.HRegion) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1717) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3076) at sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1326) "IPC Server handler 24 on 9009" daemon prio=10 tid=0x00002aaac0a7d800 nid=0x6d5f in Object.wait() [0x0000000045c6b000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:2051) - locked <0x00000006a84b9218> (a org.apache.hadoop.hbase.regionserver.HRegion) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1717) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3076) at sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1326) "IPC Server handler 23 on 9009" daemon prio=10 tid=0x00002aaac0a7b800 nid=0x6d5e in Object.wait() [0x0000000045b6a000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:2051) - locked <0x00000006a84b9218> (a org.apache.hadoop.hbase.regionserver.HRegion) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1717) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3076) at sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1326) "IPC Server handler 22 on 9009" daemon prio=10 tid=0x00002aaac055f000 nid=0x6d5d in Object.wait() [0x0000000045a69000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:2051) - locked <0x00000006a84b9218> (a org.apache.hadoop.hbase.regionserver.HRegion) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1717) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3076) at sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1326) "IPC Server handler 21 on 9009" daemon prio=10 tid=0x00002aaac07f7800 nid=0x6d5c in Object.wait() [0x0000000045968000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:2051) - locked <0x00000006a84b9218> (a org.apache.hadoop.hbase.regionserver.HRegion) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1717) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3076) at sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1326) "IPC Server handler 20 on 9009" daemon prio=10 tid=0x00002aaacc236800 nid=0x6d5b in Object.wait() [0x0000000045867000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:2051) - locked <0x00000006a84b9218> (a org.apache.hadoop.hbase.regionserver.HRegion) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1717) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3076) at sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1326) "IPC Server handler 19 on 9009" daemon prio=10 tid=0x0000000052fc7800 nid=0x6d5a in Object.wait() [0x0000000045766000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:2051) - locked <0x00000006a84b9218> (a org.apache.hadoop.hbase.regionserver.HRegion) at org.apache.hadoop.hbase.regionserver.HRegion.delete(HRegion.java:1469) at org.apache.hadoop.hbase.regionserver.HRegionServer.delete(HRegionServer.java:2260) at sun.reflect.GeneratedMethodAccessor36.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1326) "IPC Server handler 18 on 9009" daemon prio=10 tid=0x0000000052c84000 nid=0x6d59 in Object.wait() [0x0000000045665000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:2051) - locked <0x00000006a84b9218> (a org.apache.hadoop.hbase.regionserver.HRegion) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1717) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3076) at sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1326) "IPC Server handler 17 on 9009" daemon prio=10 tid=0x0000000052c82000 nid=0x6d58 in Object.wait() [0x0000000045564000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:2051) - locked <0x00000006a84b9218> (a org.apache.hadoop.hbase.regionserver.HRegion) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1717) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3076) at sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1326) "IPC Server handler 16 on 9009" daemon prio=10 tid=0x0000000052c7f800 nid=0x6d57 in Object.wait() [0x0000000045463000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:2051) - locked <0x00000006a84b9218> (a org.apache.hadoop.hbase.regionserver.HRegion) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1717) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3076) at sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1326) "IPC Server handler 15 on 9009" daemon prio=10 tid=0x0000000052a03800 nid=0x6d56 in Object.wait() [0x0000000045362000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:2051) - locked <0x00000006a84b9218> (a org.apache.hadoop.hbase.regionserver.HRegion) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1717) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3076) at sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1326) "IPC Server handler 14 on 9009" daemon prio=10 tid=0x0000000052a01000 nid=0x6d55 in Object.wait() [0x0000000045261000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:2051) - locked <0x00000006a84b9218> (a org.apache.hadoop.hbase.regionserver.HRegion) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1717) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3076) at sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1326) "IPC Server handler 13 on 9009" daemon prio=10 tid=0x00000000529ff000 nid=0x6d54 in Object.wait() [0x0000000045160000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:2051) - locked <0x00000006a84b9218> (a org.apache.hadoop.hbase.regionserver.HRegion) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1717) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3076) at sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1326) "IPC Server handler 12 on 9009" daemon prio=10 tid=0x0000000052c65800 nid=0x6d53 in Object.wait() [0x000000004505f000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:2051) - locked <0x00000006a84b9218> (a org.apache.hadoop.hbase.regionserver.HRegion) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1717) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3076) at sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1326) "IPC Server handler 11 on 9009" daemon prio=10 tid=0x0000000052c64000 nid=0x6d52 in Object.wait() [0x0000000044f5e000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:2051) - locked <0x00000006a84b9218> (a org.apache.hadoop.hbase.regionserver.HRegion) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1717) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3076) at sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1326) "IPC Server handler 10 on 9009" daemon prio=10 tid=0x0000000053170800 nid=0x6d51 in Object.wait() [0x0000000044e5d000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:2051) - locked <0x00000006a84b9218> (a org.apache.hadoop.hbase.regionserver.HRegion) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1717) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3076) at sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1326) "IPC Server handler 9 on 9009" daemon prio=10 tid=0x0000000052fd7000 nid=0x6d50 in Object.wait() [0x0000000044d5c000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:2051) - locked <0x00000006a84b9218> (a org.apache.hadoop.hbase.regionserver.HRegion) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1717) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3076) at sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1326) "IPC Server handler 8 on 9009" daemon prio=10 tid=0x0000000052e1f000 nid=0x6d4f in Object.wait() [0x0000000044c5b000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:2051) - locked <0x00000006a84b9218> (a org.apache.hadoop.hbase.regionserver.HRegion) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1717) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3076) at sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1326) "IPC Server handler 7 on 9009" daemon prio=10 tid=0x0000000052bd9000 nid=0x6d4e in Object.wait() [0x0000000044b5a000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:2051) - locked <0x00000006a84b9218> (a org.apache.hadoop.hbase.regionserver.HRegion) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1717) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3076) at sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1326) "IPC Server handler 6 on 9009" daemon prio=10 tid=0x0000000052be9000 nid=0x6d4d in Object.wait() [0x0000000044a59000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:2051) - locked <0x00000006a84b9218> (a org.apache.hadoop.hbase.regionserver.HRegion) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1717) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3076) at sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1326) "IPC Server handler 5 on 9009" daemon prio=10 tid=0x0000000053160800 nid=0x6d4c in Object.wait() [0x0000000044958000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:2051) - locked <0x00000006a84b9218> (a org.apache.hadoop.hbase.regionserver.HRegion) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1717) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3076) at sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1326) "IPC Server handler 4 on 9009" daemon prio=10 tid=0x0000000052ee3800 nid=0x6d4b in Object.wait() [0x0000000044857000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:2051) - locked <0x00000006a84b9218> (a org.apache.hadoop.hbase.regionserver.HRegion) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1717) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3076) at sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1326) "IPC Server handler 3 on 9009" daemon prio=10 tid=0x0000000052c02800 nid=0x6d4a in Object.wait() [0x0000000044756000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:2051) - locked <0x00000006a84b9218> (a org.apache.hadoop.hbase.regionserver.HRegion) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1717) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3076) at sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1326) "IPC Server handler 2 on 9009" daemon prio=10 tid=0x0000000052c01000 nid=0x6d49 in Object.wait() [0x0000000044655000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:2051) - locked <0x00000006a84b9218> (a org.apache.hadoop.hbase.regionserver.HRegion) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1717) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3076) at sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1326) "IPC Server handler 1 on 9009" daemon prio=10 tid=0x0000000052a22800 nid=0x6d48 in Object.wait() [0x0000000044554000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:2051) - locked <0x00000006a84b9218> (a org.apache.hadoop.hbase.regionserver.HRegion) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1717) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3076) at sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1326) "IPC Server handler 0 on 9009" daemon prio=10 tid=0x0000000052c19800 nid=0x6d47 in Object.wait() [0x0000000044453000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:2051) - locked <0x00000006a84b9218> (a org.apache.hadoop.hbase.regionserver.HRegion) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1717) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3076) at sun.reflect.GeneratedMethodAccessor38.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1326) "IPC Server listener on 9009" daemon prio=10 tid=0x0000000052c54800 nid=0x6d46 runnable [0x0000000044352000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) - locked <0x000000058b300e80> (a sun.nio.ch.Util$2) - locked <0x000000058b300e98> (a java.util.Collections$UnmodifiableSet) - locked <0x000000058b302f20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener.run(HBaseServer.java:592) "IPC Server Responder" daemon prio=10 tid=0x0000000052c54000 nid=0x6d45 runnable [0x0000000044251000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) - locked <0x000000058b33b220> (a sun.nio.ch.Util$2) - locked <0x000000058b33b238> (a java.util.Collections$UnmodifiableSet) - locked <0x000000058b33aba0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) at org.apache.hadoop.hbase.ipc.HBaseServer$Responder.doRunLoop(HBaseServer.java:782) at org.apache.hadoop.hbase.ipc.HBaseServer$Responder.run(HBaseServer.java:765) "Timer-0" daemon prio=10 tid=0x0000000053121000 nid=0x6d44 in Object.wait() [0x0000000044150000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:509) - locked <0x000000058b7e83a8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:462) "1861006321@qtp-1068922666-1 - Acceptor0 SelectChannelConnector@0.0.0.0:9004" prio=10 tid=0x00002aaac0079800 nid=0x6d41 runnable [0x000000004404f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) - locked <0x000000058b959868> (a sun.nio.ch.Util$2) - locked <0x000000058b959880> (a java.util.Collections$UnmodifiableSet) - locked <0x000000058b3b5400> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) at org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498) at org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192) at org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124) at org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708) at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) "regionserver9009.leaseChecker" daemon prio=10 tid=0x00002aaac002b800 nid=0x6d3f waiting on condition [0x0000000043e4d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000058b30a830> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2025) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:201) at org.apache.hadoop.hbase.regionserver.Leases.run(Leases.java:83) at java.lang.Thread.run(Thread.java:662) "regionserver9009.compactionChecker" daemon prio=10 tid=0x00002aaac005d800 nid=0x6d3e in Object.wait() [0x0000000043d4c000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:91) - locked <0x000000058b9597c0> (a java.lang.Object) at org.apache.hadoop.hbase.Chore.run(Chore.java:75) at java.lang.Thread.run(Thread.java:662) "regionserver9009.cacheFlusher" daemon prio=10 tid=0x00002aaac005d000 nid=0x6d3d waiting for monitor entry [0x0000000043c4b000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.StoreScanner.updateReaders(StoreScanner.java:360) - waiting to lock <0x000000058fd24658> (a org.apache.hadoop.hbase.regionserver.StoreScanner) at org.apache.hadoop.hbase.regionserver.Store.notifyChangedReadersObservers(Store.java:667) at org.apache.hadoop.hbase.regionserver.Store.updateStorefiles(Store.java:656) at org.apache.hadoop.hbase.regionserver.Store.access$200(Store.java:89) at org.apache.hadoop.hbase.regionserver.Store$StoreFlusherImpl.commit(Store.java:1916) at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:1259) at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:1162) at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:1104) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:400) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:374) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.run(MemStoreFlusher.java:243) at java.lang.Thread.run(Thread.java:662) "regionserver9009.logRoller" daemon prio=10 tid=0x00002aaac00ca000 nid=0x6d3c waiting on condition [0x0000000043b4a000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000058b8f29a8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:811) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:842) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:1178) at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(ReentrantLock.java:186) at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock.java:262) at org.apache.hadoop.hbase.regionserver.wal.HLog.rollWriter(HLog.java:549) at org.apache.hadoop.hbase.regionserver.LogRoller.run(LogRoller.java:94) at java.lang.Thread.run(Thread.java:662) "Timer thread for monitoring jvm" daemon prio=10 tid=0x00002aaac00c8000 nid=0x6d3b in Object.wait() [0x0000000043a49000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:509) - locked <0x000000058b9597d8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:462) "Timer thread for monitoring hbase" daemon prio=10 tid=0x00002aaac0720800 nid=0x6d3a in Object.wait() [0x0000000043948000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:509) - locked <0x000000058b959790> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:462) "regionserver9009.logSyncer" daemon prio=10 tid=0x00002aaac07ac000 nid=0x6d39 waiting on condition [0x0000000043847000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hbase.regionserver.wal.HLog$LogSyncer.run(HLog.java:1059) at java.lang.Thread.run(Thread.java:662) "LeaseChecker" daemon prio=10 tid=0x00002aaac0698000 nid=0x6d38 waiting on condition [0x00000000411c5000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.DFSClient$LeaseChecker.run(DFSClient.java:1302) at java.lang.Thread.run(Thread.java:662) "IncidentDistributionPoint-0/ProcessSyncQueue" daemon prio=10 tid=0x0000000052e99800 nid=0x6d32 in Object.wait() [0x0000000043645000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:485) at ORG.oclc.infra.util.SyncQueue.waitForEntries(SyncQueue.java:276) - locked <0x000000058b304f08> (a ORG.oclc.infra.util.SyncQueue) at ORG.oclc.infra.drax.DestinationList.execute(DestinationList.java:115) at ORG.oclc.infra.drax.DistributionPoint.run(DistributionPoint.java:134) at java.lang.Thread.run(Thread.java:662) "regionserver9009-EventThread" daemon prio=10 tid=0x0000000052f61000 nid=0x6cfc waiting on condition [0x0000000040df7000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000058b300b40> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:493) "regionserver9009-SendThread(wchddb35pxdu.prod.oclc.org:9011)" daemon prio=10 tid=0x000000005259e000 nid=0x6cfa runnable [0x0000000040bf5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) - locked <0x000000058b300a38> (a sun.nio.ch.Util$2) - locked <0x000000058b300a50> (a java.util.Collections$UnmodifiableSet) - locked <0x000000058b3068d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:274) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1035) "DestroyJavaVM" prio=10 tid=0x00002aaac0ab5000 nid=0x6b6a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "regionserver9009-EventThread" daemon prio=10 tid=0x00002aaacc165800 nid=0x6bbc waiting on condition [0x0000000040af4000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000058b300b88> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:493) "regionserver9009-SendThread(wchddb02pxdu.prod.oclc.org:9011)" daemon prio=10 tid=0x00002aaacc162800 nid=0x6bbb runnable [0x000000004032e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) - locked <0x000000058b300bd0> (a sun.nio.ch.Util$2) - locked <0x000000058b300be8> (a java.util.Collections$UnmodifiableSet) - locked <0x000000058b300858> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:274) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1035) "regionserver9009" prio=10 tid=0x00000000530ad000 nid=0x6bb9 in Object.wait() [0x0000000043241000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:91) - locked <0x000000058b300a98> (a java.lang.Object) at org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:55) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:691) at java.lang.Thread.run(Thread.java:662) "LRU Statistics #0" daemon prio=10 tid=0x00000000530d8800 nid=0x6bb8 waiting on condition [0x0000000043140000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000058b300c30> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2025) at java.util.concurrent.DelayQueue.take(DelayQueue.java:164) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:609) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:602) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:662) "LruBlockCache.EvictionThread" daemon prio=10 tid=0x00000000528c3000 nid=0x6bb7 in Object.wait() [0x0000000041c3d000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:485) at org.apache.hadoop.hbase.io.hfile.LruBlockCache$EvictionThread.run(LruBlockCache.java:566) - locked <0x000000058b300ab0> (a org.apache.hadoop.hbase.io.hfile.LruBlockCache$EvictionThread) at java.lang.Thread.run(Thread.java:662) "Timer thread for monitoring rpc" daemon prio=10 tid=0x0000000052f48000 nid=0x6bb3 in Object.wait() [0x000000004303f000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:509) - locked <0x000000058b300ee0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:462) "IPC Reader 9 on port 9009" daemon prio=10 tid=0x0000000052b5d000 nid=0x6bb2 runnable [0x0000000042f3e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) - locked <0x000000058b300ac8> (a sun.nio.ch.Util$2) - locked <0x000000058b300ae0> (a java.util.Collections$UnmodifiableSet) - locked <0x000000058b306920> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:484) - locked <0x000000058b300af8> (a org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:470) at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) at java.lang.Thread.run(Thread.java:662) "IPC Reader 8 on port 9009" daemon prio=10 tid=0x0000000052b5b000 nid=0x6bb1 runnable [0x0000000042e3d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) - locked <0x000000058b300028> (a sun.nio.ch.Util$2) - locked <0x000000058b300040> (a java.util.Collections$UnmodifiableSet) - locked <0x000000058b30a9b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:484) - locked <0x000000058b300058> (a org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:470) at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) at java.lang.Thread.run(Thread.java:662) "IPC Reader 7 on port 9009" daemon prio=10 tid=0x0000000052a30800 nid=0x6bb0 runnable [0x0000000042d3c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) - locked <0x000000058b300f10> (a sun.nio.ch.Util$2) - locked <0x000000058b300f28> (a java.util.Collections$UnmodifiableSet) - locked <0x000000058b302f68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:484) - locked <0x000000058b300f40> (a org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:470) at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) at java.lang.Thread.run(Thread.java:662) "IPC Reader 6 on port 9009" daemon prio=10 tid=0x0000000052a2e800 nid=0x6bae runnable [0x0000000042c3b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) - locked <0x000000058b30ab18> (a sun.nio.ch.Util$2) - locked <0x000000058b30ab30> (a java.util.Collections$UnmodifiableSet) - locked <0x000000058b302fb0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:484) - locked <0x000000058b30ab48> (a org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:470) at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) at java.lang.Thread.run(Thread.java:662) "IPC Reader 5 on port 9009" daemon prio=10 tid=0x0000000053128800 nid=0x6bad runnable [0x0000000042b3a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) - locked <0x000000058b300088> (a sun.nio.ch.Util$2) - locked <0x000000058b3000a0> (a java.util.Collections$UnmodifiableSet) - locked <0x000000058b30a9f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:484) - locked <0x000000058b3000b8> (a org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:470) at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) at java.lang.Thread.run(Thread.java:662) "IPC Reader 4 on port 9009" daemon prio=10 tid=0x0000000052c86800 nid=0x6bac runnable [0x0000000042a39000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) - locked <0x000000058b30a878> (a sun.nio.ch.Util$2) - locked <0x000000058b30a890> (a java.util.Collections$UnmodifiableSet) - locked <0x000000058b30b738> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:484) - locked <0x000000058b30a8a8> (a org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:470) at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) at java.lang.Thread.run(Thread.java:662) "IPC Reader 3 on port 9009" daemon prio=10 tid=0x0000000052f51800 nid=0x6bab runnable [0x0000000042938000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) - locked <0x000000058b30ab78> (a sun.nio.ch.Util$2) - locked <0x000000058b30ab90> (a java.util.Collections$UnmodifiableSet) - locked <0x000000058b302ff8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:484) - locked <0x000000058b30aba8> (a org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:470) at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) at java.lang.Thread.run(Thread.java:662) "IPC Reader 2 on port 9009" daemon prio=10 tid=0x00000000528fc800 nid=0x6baa runnable [0x0000000042837000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) - locked <0x000000058b3000e8> (a sun.nio.ch.Util$2) - locked <0x000000058b300100> (a java.util.Collections$UnmodifiableSet) - locked <0x000000058b30aa40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:484) - locked <0x000000058b300118> (a org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:470) at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) at java.lang.Thread.run(Thread.java:662) "IPC Reader 1 on port 9009" daemon prio=10 tid=0x0000000052ecb000 nid=0x6ba9 runnable [0x0000000042736000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) - locked <0x000000058b300b28> (a sun.nio.ch.Util$2) - locked <0x000000058b30c0f8> (a java.util.Collections$UnmodifiableSet) - locked <0x000000058b306968> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:484) - locked <0x000000058b30c110> (a org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:470) at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) at java.lang.Thread.run(Thread.java:662) "IPC Reader 0 on port 9009" daemon prio=10 tid=0x0000000052c8a000 nid=0x6ba8 runnable [0x0000000042635000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) - locked <0x000000058b30c278> (a sun.nio.ch.Util$2) - locked <0x000000058b30c260> (a java.util.Collections$UnmodifiableSet) - locked <0x000000058b303088> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:484) - locked <0x000000058b74e918> (a org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:470) at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) at java.lang.Thread.run(Thread.java:662) "RMI TCP Accept-0" daemon prio=10 tid=0x00002aaac065e000 nid=0x6b83 runnable [0x0000000040fc3000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.PlainSocketImpl.accept(PlainSocketImpl.java:408) - locked <0x000000058b30bae0> (a java.net.SocksSocketImpl) at java.net.ServerSocket.implAccept(ServerSocket.java:462) at java.net.ServerSocket.accept(ServerSocket.java:430) at sun.management.jmxremote.LocalRMIServerSocketFactory$1.accept(LocalRMIServerSocketFactory.java:34) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:369) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:341) at java.lang.Thread.run(Thread.java:662) "RMI TCP Accept-9021" daemon prio=10 tid=0x00002aaac065a800 nid=0x6b82 runnable [0x00000000409f3000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.PlainSocketImpl.accept(PlainSocketImpl.java:408) - locked <0x000000058b30c3e0> (a java.net.SocksSocketImpl) at java.net.ServerSocket.implAccept(ServerSocket.java:462) at java.net.ServerSocket.accept(ServerSocket.java:430) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:369) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:341) at java.lang.Thread.run(Thread.java:662) "RMI TCP Accept-0" daemon prio=10 tid=0x00002aaac0648000 nid=0x6b81 runnable [0x00000000408f2000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.PlainSocketImpl.accept(PlainSocketImpl.java:408) - locked <0x000000058b30ca70> (a java.net.SocksSocketImpl) at java.net.ServerSocket.implAccept(ServerSocket.java:462) at java.net.ServerSocket.accept(ServerSocket.java:430) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:369) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:341) at java.lang.Thread.run(Thread.java:662) "Low Memory Detector" daemon prio=10 tid=0x00002aaac0026800 nid=0x6b7d runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread1" daemon prio=10 tid=0x00002aaac0024000 nid=0x6b7c waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread0" daemon prio=10 tid=0x00002aaac0022000 nid=0x6b7b waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Signal Dispatcher" daemon prio=10 tid=0x00002aaac001f800 nid=0x6b7a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Surrogate Locker Thread (Concurrent GC)" daemon prio=10 tid=0x00002aaac001d800 nid=0x6b79 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Finalizer" daemon prio=10 tid=0x00002aaac0001000 nid=0x6b78 in Object.wait() [0x000000004171c000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:118) - locked <0x000000058b300160> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:134) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:159) "Reference Handler" daemon prio=10 tid=0x00000000527e3000 nid=0x6b77 in Object.wait() [0x000000004161b000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:485) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:116) - locked <0x000000058b303988> (a java.lang.ref.Reference$Lock) "VM Thread" prio=10 tid=0x00000000527dc800 nid=0x6b76 runnable "Gang worker#0 (Parallel GC Threads)" prio=10 tid=0x00000000525ac800 nid=0x6b6b runnable "Gang worker#1 (Parallel GC Threads)" prio=10 tid=0x00000000525ae800 nid=0x6b6c runnable "Gang worker#2 (Parallel GC Threads)" prio=10 tid=0x00000000525b0800 nid=0x6b6d runnable "Gang worker#3 (Parallel GC Threads)" prio=10 tid=0x00000000525b2000 nid=0x6b6e runnable "Gang worker#4 (Parallel GC Threads)" prio=10 tid=0x00000000525b4000 nid=0x6b6f runnable "Gang worker#5 (Parallel GC Threads)" prio=10 tid=0x00000000525b6000 nid=0x6b70 runnable "Gang worker#6 (Parallel GC Threads)" prio=10 tid=0x00000000525b7800 nid=0x6b71 runnable "Gang worker#7 (Parallel GC Threads)" prio=10 tid=0x00000000525b9800 nid=0x6b72 runnable "Concurrent Mark-Sweep GC Thread" prio=10 tid=0x00000000526ad800 nid=0x6b75 runnable "Gang worker#0 (Parallel CMS Threads)" prio=10 tid=0x00000000526a9800 nid=0x6b73 runnable "Gang worker#1 (Parallel CMS Threads)" prio=10 tid=0x00000000526ab000 nid=0x6b74 runnable "VM Periodic Task Thread" prio=10 tid=0x00002aaac0661800 nid=0x6b84 waiting on condition JNI global references: 1248 Heap par new generation total 235968K, used 176200K [0x000000057ae00000, 0x000000058ae00000, 0x000000058ae00000) eden space 209792K, 72% used [0x000000057ae00000, 0x0000000584214c88, 0x0000000587ae0000) from space 26176K, 93% used [0x0000000587ae0000, 0x00000005892dd6c8, 0x0000000589470000) to space 26176K, 0% used [0x0000000589470000, 0x0000000589470000, 0x000000058ae00000) concurrent mark-sweep generation total 10223616K, used 4535508K [0x000000058ae00000, 0x00000007fae00000, 0x00000007fae00000) concurrent-mark-sweep perm gen total 49196K, used 29423K [0x00000007fae00000, 0x00000007fde0b000, 0x0000000800000000) -PROD- prodcon@wchddb37pxdu:/prod/hadoop/prod/hbase-0.92.1/logs$