Want more features on Pastebin? Sign Up, it's FREE!
Guest

Hbase Replication Issue

By: a guest on Apr 18th, 2013  |  syntax: None  |  size: 7.90 KB  |  views: 13  |  expires: Never
download  |  raw  |  embed  |  report abuse  |  print
Text below is selected. Please press Ctrl+C to copy to your clipboard. (⌘+C on Mac)
  1. Starting Replication:
  2.  
  3.  
  4. 2013-04-18 01:47:33,423 INFO org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceManager: Atomically moving relevance-hbase5-snc1.snc1,60020,1366247910200's hlogs to my queue
  5. 2013-04-18 01:47:33,424 DEBUG org.apache.hadoop.hbase.replication.ReplicationZookeeper:  The multi list size is: 1
  6. 2013-04-18 01:47:33,425 WARN org.apache.hadoop.hbase.replication.ReplicationZookeeper: Got exception in copyQueuesFromRSUsingMulti:
  7. org.apache.zookeeper.KeeperException$NotEmptyException: KeeperErrorCode = Directory not empty
  8.         at org.apache.zookeeper.KeeperException.create(KeeperException.java:125)
  9.         at org.apache.zookeeper.ZooKeeper.multiInternal(ZooKeeper.java:925)
  10.         at org.apache.zookeeper.ZooKeeper.multi(ZooKeeper.java:901)
  11.         at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.multi(RecoverableZooKeeper.java:538)
  12.         at org.apache.hadoop.hbase.zookeeper.ZKUtil.multiOrSequential(ZKUtil.java:1457)
  13.         at org.apache.hadoop.hbase.replication.ReplicationZookeeper.copyQueuesFromRSUsingMulti(ReplicationZookeeper.java:705)
  14.         at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceManager$NodeFailoverWorker.run(ReplicationSourceManager.java:585)
  15.         at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
  16.         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
  17.         at java.lang.Thread.run(Thread.java:662)
  18.  
  19.  
  20. 66231012980-relevance-hbase2-snc1.snc1,60020,1366236326732-relevance-hbase2-snc1.snc1,60020,1366236715108 Got:
  21. java.io.EOFException
  22.         at java.io.DataInputStream.readFully(DataInputStream.java:180)
  23.         at java.io.DataInputStream.readFully(DataInputStream.java:152)
  24.         at org.apache.hadoop.io.SequenceFile$Reader.init(SequenceFile.java:1781)
  25.         at org.apache.hadoop.io.SequenceFile$Reader.initialize(SequenceFile.java:1746)
  26.         at org.apache.hadoop.io.SequenceFile$Reader.<init>(SequenceFile.java:1695)
  27.         at org.apache.hadoop.io.SequenceFile$Reader.<init>(SequenceFile.java:1709)
  28.         at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogReader$WALReader.<init>(SequenceFileLogReader.java:55)
  29.         at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogReader.init(SequenceFileLogReader.java:177)
  30.         at org.apache.hadoop.hbase.regionserver.wal.HLog.getReader(HLog.java:720)
  31.         at org.apache.hadoop.hbase.replication.regionserver.ReplicationHLogReaderManager.openReader(ReplicationHLogReaderManager.java:66)
  32.         at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.openReader(ReplicationSource.java:501)
  33.         at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.run(ReplicationSource.java:312)
  34.  
  35.  
  36. org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /sd_relevance_hbase/replication/rs/relevance-hbase2-snc1.snc1,60020,1366247745434/2-relevance-hbase9-snc1.snc1,60020,1366246895889-relevance-hbase10-snc1.snc1,60020,1366246951564-relevance-hbase2-snc1.snc1,60020,1366247621091/relevance-hbase9-snc1.snc1%2C60020%2C1366246895889.1366246896827
  37.         at org.apache.zookeeper.KeeperException.create(KeeperException.java:111)
  38.         at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
  39.         at org.apache.zookeeper.ZooKeeper.setData(ZooKeeper.java:1246)
  40.         at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.setData(RecoverableZooKeeper.java:354)
  41.         at org.apache.hadoop.hbase.zookeeper.ZKUtil.setData(ZKUtil.java:846)
  42.         at org.apache.hadoop.hbase.zookeeper.ZKUtil.setData(ZKUtil.java:898)
  43.         at org.apache.hadoop.hbase.zookeeper.ZKUtil.setData(ZKUtil.java:892)
  44.         at org.apache.hadoop.hbase.replication.ReplicationZookeeper.writeReplicationStatus(ReplicationZookeeper.java:558)
  45.         at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceManager.logPositionAndCleanOldLogs(ReplicationSourceManager.java:154)
  46.         at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.run(ReplicationSource.java:376)
  47. 2013-04-18 01:47:36,043 FATAL org.apache.hadoop.hbase.regionserver.HRegionServer: ABORTING region server relevance-hbase2-snc1.snc1,60020,1366247745434: Writing replication status
  48. 2013-04-18 01:47:39,485 WARN org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper: Possibly transient ZooKeeper exception: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /sd_relevance_hbase/replication/rs/relevance-hbase2-snc1.snc1,60020,1366247745434/2-relevance-hbase8-snc1.snc1,60020,1366226193038-relevance-hbase2-snc1.snc1,60020,1366231012980-relevance-hbase2-snc1.snc1,60020,1366236326732-relevance-hbase2-snc1.snc1,60020,1366236715108/relevance-hbase8-snc1.snc1%2C60020%2C1366226193038.1366234122705
  49. 2013-04-18 01:47:39,485 INFO org.apache.hadoop.hbase.util.RetryCounter: Sleeping 2000ms before retry #1...
  50. 2013-04-18 01:47:39,559 INFO org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceManager: Not transferring queue since we are shutting down
  51. 2013-04-18 01:47:40,663 WARN org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper: Possibly transient ZooKeeper exception: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /sd_relevance_hbase/splitlog/hdfs%3A%2F%2F10.20.74.126%2Frelevance_hbase%2F.logs%2Frelevance-hbase5-snc1.snc1%2C60020%2C1366247910200-splitting%2Frelevance-hbase5-snc1.snc1%252C60020%252C1366247910200.1366249417827
  52. 2013-04-18 01:47:40,664 INFO org.apache.hadoop.hbase.util.RetryCounter: Sleeping 4000ms before retry #2...
  53. 2013-04-18 01:47:41,293 WARN org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper: Possibly transient ZooKeeper exception: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /sd_relevance_hbase/replication/rs/relevance-hbase2-snc1.snc1,60020,1366247745434/2-relevance-hbase2-snc1.snc1,60020,1366231012980-relevance-hbase2-snc1.snc1,60020,1366236326732-relevance-hbase2-snc1.snc1,60020,1366236715108/relevance-hbase2-snc1.snc1%2C60020%2C1366231012980.1366233878323
  54.  
  55. ERROR org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper: ZooKeeper setData failed after 3 retries
  56. 2013-04-18 01:47:55,302 FATAL org.apache.hadoop.hbase.regionserver.HRegionServer: ABORTING region server relevance-hbase2-snc1.snc1,60020,1366247745434: Writing replication status
  57. org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /sd_relevance_hbase/replication/rs/relevance-hbase2-snc1.snc1,60020,1366247745434/2-relevance-hbase2-snc1.snc1,60020,1366231012980-relevance-hbase2-snc1.snc1,60020,1366236326732-relevance-hbase2-snc1.snc1,60020,1366236715108/relevance-hbase2-snc1.snc1%2C60020%2C1366231012980.1366233878323
  58.         at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
  59.         at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
  60.         at org.apache.zookeeper.ZooKeeper.setData(ZooKeeper.java:1246)
  61.         at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.setData(RecoverableZooKeeper.java:354)
  62.         at org.apache.hadoop.hbase.zookeeper.ZKUtil.setData(ZKUtil.java:846)
  63.         at org.apache.hadoop.hbase.zookeeper.ZKUtil.setData(ZKUtil.java:898)
  64.         at org.apache.hadoop.hbase.zookeeper.ZKUtil.setData(ZKUtil.java:892)
  65.         at org.apache.hadoop.hbase.replication.ReplicationZookeeper.writeReplicationStatus(ReplicationZookeeper.java:558)
  66.         at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceManager.logPositionAndCleanOldLogs(ReplicationSourceManager.java:154)
  67.         at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.shipEdits(ReplicationSource.java:638)
  68.         at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.run(ReplicationSource.java:387)
  69. 2013-04-18 01:47:55,302 FATAL org.apache.hadoop.hbase.regionserver.HRegionServer: RegionServer abort: loaded coprocessors are: []
clone this paste RAW Paste Data