Starting Replication:
2013-04-18 01:47:33,423 INFO org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceManager: Atomically moving relevance-hbase5-snc1.snc1,60020,1366247910200's hlogs to my queue
2013-04-18 01:47:33,424 DEBUG org.apache.hadoop.hbase.replication.ReplicationZookeeper: The multi list size is: 1
2013-04-18 01:47:33,425 WARN org.apache.hadoop.hbase.replication.ReplicationZookeeper: Got exception in copyQueuesFromRSUsingMulti:
org.apache.zookeeper.KeeperException$NotEmptyException: KeeperErrorCode = Directory not empty
at org.apache.zookeeper.KeeperException.create(KeeperException.java:125)
at org.apache.zookeeper.ZooKeeper.multiInternal(ZooKeeper.java:925)
at org.apache.zookeeper.ZooKeeper.multi(ZooKeeper.java:901)
at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.multi(RecoverableZooKeeper.java:538)
at org.apache.hadoop.hbase.zookeeper.ZKUtil.multiOrSequential(ZKUtil.java:1457)
at org.apache.hadoop.hbase.replication.ReplicationZookeeper.copyQueuesFromRSUsingMulti(ReplicationZookeeper.java:705)
at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceManager$NodeFailoverWorker.run(ReplicationSourceManager.java:585)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:662)
66231012980-relevance-hbase2-snc1.snc1,60020,1366236326732-relevance-hbase2-snc1.snc1,60020,1366236715108 Got:
java.io.EOFException
at java.io.DataInputStream.readFully(DataInputStream.java:180)
at java.io.DataInputStream.readFully(DataInputStream.java:152)
at org.apache.hadoop.io.SequenceFile$Reader.init(SequenceFile.java:1781)
at org.apache.hadoop.io.SequenceFile$Reader.initialize(SequenceFile.java:1746)
at org.apache.hadoop.io.SequenceFile$Reader.<init>(SequenceFile.java:1695)
at org.apache.hadoop.io.SequenceFile$Reader.<init>(SequenceFile.java:1709)
at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogReader$WALReader.<init>(SequenceFileLogReader.java:55)
at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogReader.init(SequenceFileLogReader.java:177)
at org.apache.hadoop.hbase.regionserver.wal.HLog.getReader(HLog.java:720)
at org.apache.hadoop.hbase.replication.regionserver.ReplicationHLogReaderManager.openReader(ReplicationHLogReaderManager.java:66)
at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.openReader(ReplicationSource.java:501)
at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.run(ReplicationSource.java:312)
org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /sd_relevance_hbase/replication/rs/relevance-hbase2-snc1.snc1,60020,1366247745434/2-relevance-hbase9-snc1.snc1,60020,1366246895889-relevance-hbase10-snc1.snc1,60020,1366246951564-relevance-hbase2-snc1.snc1,60020,1366247621091/relevance-hbase9-snc1.snc1%2C60020%2C1366246895889.1366246896827
at org.apache.zookeeper.KeeperException.create(KeeperException.java:111)
at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
at org.apache.zookeeper.ZooKeeper.setData(ZooKeeper.java:1246)
at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.setData(RecoverableZooKeeper.java:354)
at org.apache.hadoop.hbase.zookeeper.ZKUtil.setData(ZKUtil.java:846)
at org.apache.hadoop.hbase.zookeeper.ZKUtil.setData(ZKUtil.java:898)
at org.apache.hadoop.hbase.zookeeper.ZKUtil.setData(ZKUtil.java:892)
at org.apache.hadoop.hbase.replication.ReplicationZookeeper.writeReplicationStatus(ReplicationZookeeper.java:558)
at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceManager.logPositionAndCleanOldLogs(ReplicationSourceManager.java:154)
at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.run(ReplicationSource.java:376)
2013-04-18 01:47:36,043 FATAL org.apache.hadoop.hbase.regionserver.HRegionServer: ABORTING region server relevance-hbase2-snc1.snc1,60020,1366247745434: Writing replication status
2013-04-18 01:47:39,485 WARN org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper: Possibly transient ZooKeeper exception: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /sd_relevance_hbase/replication/rs/relevance-hbase2-snc1.snc1,60020,1366247745434/2-relevance-hbase8-snc1.snc1,60020,1366226193038-relevance-hbase2-snc1.snc1,60020,1366231012980-relevance-hbase2-snc1.snc1,60020,1366236326732-relevance-hbase2-snc1.snc1,60020,1366236715108/relevance-hbase8-snc1.snc1%2C60020%2C1366226193038.1366234122705
2013-04-18 01:47:39,485 INFO org.apache.hadoop.hbase.util.RetryCounter: Sleeping 2000ms before retry #1...
2013-04-18 01:47:39,559 INFO org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceManager: Not transferring queue since we are shutting down
2013-04-18 01:47:40,663 WARN org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper: Possibly transient ZooKeeper exception: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /sd_relevance_hbase/splitlog/hdfs%3A%2F%2F10.20.74.126%2Frelevance_hbase%2F.logs%2Frelevance-hbase5-snc1.snc1%2C60020%2C1366247910200-splitting%2Frelevance-hbase5-snc1.snc1%252C60020%252C1366247910200.1366249417827
2013-04-18 01:47:40,664 INFO org.apache.hadoop.hbase.util.RetryCounter: Sleeping 4000ms before retry #2...
2013-04-18 01:47:41,293 WARN org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper: Possibly transient ZooKeeper exception: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /sd_relevance_hbase/replication/rs/relevance-hbase2-snc1.snc1,60020,1366247745434/2-relevance-hbase2-snc1.snc1,60020,1366231012980-relevance-hbase2-snc1.snc1,60020,1366236326732-relevance-hbase2-snc1.snc1,60020,1366236715108/relevance-hbase2-snc1.snc1%2C60020%2C1366231012980.1366233878323
ERROR org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper: ZooKeeper setData failed after 3 retries
2013-04-18 01:47:55,302 FATAL org.apache.hadoop.hbase.regionserver.HRegionServer: ABORTING region server relevance-hbase2-snc1.snc1,60020,1366247745434: Writing replication status
org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /sd_relevance_hbase/replication/rs/relevance-hbase2-snc1.snc1,60020,1366247745434/2-relevance-hbase2-snc1.snc1,60020,1366231012980-relevance-hbase2-snc1.snc1,60020,1366236326732-relevance-hbase2-snc1.snc1,60020,1366236715108/relevance-hbase2-snc1.snc1%2C60020%2C1366231012980.1366233878323
at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
at org.apache.zookeeper.ZooKeeper.setData(ZooKeeper.java:1246)
at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.setData(RecoverableZooKeeper.java:354)
at org.apache.hadoop.hbase.zookeeper.ZKUtil.setData(ZKUtil.java:846)
at org.apache.hadoop.hbase.zookeeper.ZKUtil.setData(ZKUtil.java:898)
at org.apache.hadoop.hbase.zookeeper.ZKUtil.setData(ZKUtil.java:892)
at org.apache.hadoop.hbase.replication.ReplicationZookeeper.writeReplicationStatus(ReplicationZookeeper.java:558)
at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceManager.logPositionAndCleanOldLogs(ReplicationSourceManager.java:154)
at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.shipEdits(ReplicationSource.java:638)
at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.run(ReplicationSource.java:387)
2013-04-18 01:47:55,302 FATAL org.apache.hadoop.hbase.regionserver.HRegionServer: RegionServer abort: loaded coprocessors are: []