Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- Wed Nov 09 00:51:58 UTC 2016, RpcRetryingCaller{globalStartTime=1478652206658, pause=100, retries=35}, org.apache.hadoop.hbase.regionserver.UnexpectedStateException: org.apache.hadoop.hbase.regionserver.UnexpectedStateException: Current snapshot id is -1,passed 1478639480535
- at org.apache.hadoop.hbase.regionserver.DefaultMemStore.clearSnapshot(DefaultMemStore.java:191)
- at org.apache.hadoop.hbase.regionserver.HStore.updateStorefiles(HStore.java:1082)
- at org.apache.hadoop.hbase.regionserver.HStore.access$600(HStore.java:119)
- at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.replayFlush(HStore.java:2377)
- at org.apache.hadoop.hbase.regionserver.HRegion.replayFlushInStores(HRegion.java:4565)
- at org.apache.hadoop.hbase.regionserver.HRegion.replayWALFlushCommitMarker(HRegion.java:4471)
- at org.apache.hadoop.hbase.regionserver.HRegion.replayWALFlushMarker(HRegion.java:4272)
- at org.apache.hadoop.hbase.regionserver.RSRpcServices.doReplayBatchOp(RSRpcServices.java:835)
- at org.apache.hadoop.hbase.regionserver.RSRpcServices.replay(RSRpcServices.java:1765)
- at org.apache.hadoop.hbase.protobuf.generated.AdminProtos$AdminService$2.callBlockingMethod(AdminProtos.java:22255)
- at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2170)
- at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:109)
- at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:133)
- at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:108)
- at java.lang.Thread.run(Thread.java:745)
- Wed Nov 09 00:52:19 UTC 2016, RpcRetryingCaller{globalStartTime=1478652206658, pause=100, retries=35}, org.apache.hadoop.hbase.regionserver.UnexpectedStateException: org.apache.hadoop.hbase.regionserver.UnexpectedStateException: Current snapshot id is -1,passed 1478639480535
- at org.apache.hadoop.hbase.regionserver.DefaultMemStore.clearSnapshot(DefaultMemStore.java:191)
- at org.apache.hadoop.hbase.regionserver.HStore.updateStorefiles(HStore.java:1082)
- at org.apache.hadoop.hbase.regionserver.HStore.access$600(HStore.java:119)
- at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.replayFlush(HStore.java:2377)
- at org.apache.hadoop.hbase.regionserver.HRegion.replayFlushInStores(HRegion.java:4565)
- at org.apache.hadoop.hbase.regionserver.HRegion.replayWALFlushCommitMarker(HRegion.java:4471)
- at org.apache.hadoop.hbase.regionserver.HRegion.replayWALFlushMarker(HRegion.java:4272)
- at org.apache.hadoop.hbase.regionserver.RSRpcServices.doReplayBatchOp(RSRpcServices.java:835)
- at org.apache.hadoop.hbase.regionserver.RSRpcServices.replay(RSRpcServices.java:1765)
- at org.apache.hadoop.hbase.protobuf.generated.AdminProtos$AdminService$2.callBlockingMethod(AdminProtos.java:22255)
- at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2170)
- at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:109)
- at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:133)
- at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:108)
- at java.lang.Thread.run(Thread.java:745)
- Wed Nov 09 00:52:39 UTC 2016, RpcRetryingCaller{globalStartTime=1478652206658, pause=100, retries=35}, org.apache.hadoop.hbase.regionserver.UnexpectedStateException: org.apache.hadoop.hbase.regionserver.UnexpectedStateException: Current snapshot id is -1,passed 1478639480535
- at org.apache.hadoop.hbase.regionserver.DefaultMemStore.clearSnapshot(DefaultMemStore.java:191)
- at org.apache.hadoop.hbase.regionserver.HStore.updateStorefiles(HStore.java:1082)
- at org.apache.hadoop.hbase.regionserver.HStore.access$600(HStore.java:119)
- at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.replayFlush(HStore.java:2377)
- at org.apache.hadoop.hbase.regionserver.HRegion.replayFlushInStores(HRegion.java:4565)
- at org.apache.hadoop.hbase.regionserver.HRegion.replayWALFlushCommitMarker(HRegion.java:4471)
- at org.apache.hadoop.hbase.regionserver.HRegion.replayWALFlushMarker(HRegion.java:4272)
- at org.apache.hadoop.hbase.regionserver.RSRpcServices.doReplayBatchOp(RSRpcServices.java:835)
- at org.apache.hadoop.hbase.regionserver.RSRpcServices.replay(RSRpcServices.java:1765)
- at org.apache.hadoop.hbase.protobuf.generated.AdminProtos$AdminService$2.callBlockingMethod(AdminProtos.java:22255)
- at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2170)
- at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:109)
- at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:133)
- at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:108)
- at java.lang.Thread.run(Thread.java:745)
- at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:147)
- at org.apache.hadoop.hbase.replication.regionserver.RegionReplicaReplicationEndpoint$RetryingRpcCallable.call(RegionReplicaReplicationEndpoint.java:606)
- at java.util.concurrent.FutureTask.run(FutureTask.java:266)
- at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
- at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
- at java.lang.Thread.run(Thread.java:745)
- Caused by: org.apache.hadoop.hbase.regionserver.UnexpectedStateException: org.apache.hadoop.hbase.regionserver.UnexpectedStateException: Current snapshot id is -1,passed 1478639480535
- at org.apache.hadoop.hbase.regionserver.DefaultMemStore.clearSnapshot(DefaultMemStore.java:191)
- at org.apache.hadoop.hbase.regionserver.HStore.updateStorefiles(HStore.java:1082)
- at org.apache.hadoop.hbase.regionserver.HStore.access$600(HStore.java:119)
- at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.replayFlush(HStore.java:2377)
- at org.apache.hadoop.hbase.regionserver.HRegion.replayFlushInStores(HRegion.java:4565)
- at org.apache.hadoop.hbase.regionserver.HRegion.replayWALFlushCommitMarker(HRegion.java:4471)
- at org.apache.hadoop.hbase.regionserver.HRegion.replayWALFlushMarker(HRegion.java:4272)
- at org.apache.hadoop.hbase.regionserver.RSRpcServices.doReplayBatchOp(RSRpcServices.java:835)
- at org.apache.hadoop.hbase.regionserver.RSRpcServices.replay(RSRpcServices.java:1765)
- at org.apache.hadoop.hbase.protobuf.generated.AdminProtos$AdminService$2.callBlockingMethod(AdminProtos.java:22255)
- at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2170)
- at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:109)
- at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:133)
- at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:108)
- at java.lang.Thread.run(Thread.java:745)
- at sun.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source)
- at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
- at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
- at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:106)
- at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:95)
- at org.apache.hadoop.hbase.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:327)
- at org.apache.hadoop.hbase.replication.regionserver.RegionReplicaReplicationEndpoint$RegionReplicaReplayCallable.replayToServer(RegionReplicaReplicationEndpoint.java:661)
- at org.apache.hadoop.hbase.replication.regionserver.RegionReplicaReplicationEndpoint$RegionReplicaReplayCallable.call(RegionReplicaReplicationEndpoint.java:633)
- at org.apache.hadoop.hbase.replication.regionserver.RegionReplicaReplicationEndpoint$RegionReplicaReplayCallable.call(RegionReplicaReplicationEndpoint.java:614)
- at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:126)
- ... 5 more
- Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.regionserver.UnexpectedStateException): org.apache.hadoop.hbase.regionserver.UnexpectedStateException: Current snapshot id is -1,passed 1478639480535
- at org.apache.hadoop.hbase.regionserver.DefaultMemStore.clearSnapshot(DefaultMemStore.java:191)
- at org.apache.hadoop.hbase.regionserver.HStore.updateStorefiles(HStore.java:1082)
- at org.apache.hadoop.hbase.regionserver.HStore.access$600(HStore.java:119)
- at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.replayFlush(HStore.java:2377)
- at org.apache.hadoop.hbase.regionserver.HRegion.replayFlushInStores(HRegion.java:4565)
- at org.apache.hadoop.hbase.regionserver.HRegion.replayWALFlushCommitMarker(HRegion.java:4471)
- at org.apache.hadoop.hbase.regionserver.HRegion.replayWALFlushMarker(HRegion.java:4272)
- at org.apache.hadoop.hbase.regionserver.RSRpcServices.doReplayBatchOp(RSRpcServices.java:835)
- at org.apache.hadoop.hbase.regionserver.RSRpcServices.replay(RSRpcServices.java:1765)
- at org.apache.hadoop.hbase.protobuf.generated.AdminProtos$AdminService$2.callBlockingMethod(AdminProtos.java:22255)
- at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2170)
- at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:109)
- at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:133)
- at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:108)
- at java.lang.Thread.run(Thread.java:745)
- at org.apache.hadoop.hbase.ipc.RpcClientImpl.call(RpcClientImpl.java:1268)
- at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:226)
- at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:331)
- at org.apache.hadoop.hbase.protobuf.generated.AdminProtos$AdminService$BlockingStub.replay(AdminProtos.java:23221)
- at org.apache.hadoop.hbase.replication.regionserver.RegionReplicaReplicationEndpoint$RegionReplicaReplayCallable.replayToServer(RegionReplicaReplicationEndpoint.java:659)
- ... 8 more
- ^C
- tim@staging-core-data-23d:~$ ^C
- tim@staging-core-data-23d:~$ head -n 20 /mnt/xvdo/var/log/hbase/hbase-cmf-hbase-REGIONSERVER-staging-core-data-23d.log.out
- 2016-11-09 18:37:48,808 WARN org.apache.hadoop.hbase.replication.regionserver.RegionReplicaReplicationEndpoint: Received IOException while trying to replicatejava.io.IOException: org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=35, exceptions:
- Wed Nov 09 00:43:26 UTC 2016, RpcRetryingCaller{globalStartTime=1478652206658, pause=100, retries=35}, org.apache.hadoop.hbase.regionserver.UnexpectedStateException: org.apache.hadoop.hbase.regionserver.UnexpectedStateException: Current snapshot id is -1,passed 1478639480535
- at org.apache.hadoop.hbase.regionserver.DefaultMemStore.clearSnapshot(DefaultMemStore.java:191)
- at org.apache.hadoop.hbase.regionserver.HStore.updateStorefiles(HStore.java:1082)
- at org.apache.hadoop.hbase.regionserver.HStore.access$600(HStore.java:119)
- at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.replayFlush(HStore.java:2377)
- at org.apache.hadoop.hbase.regionserver.HRegion.replayFlushInStores(HRegion.java:4565)
- at org.apache.hadoop.hbase.regionserver.HRegion.replayWALFlushCommitMarker(HRegion.java:4471)
- at org.apache.hadoop.hbase.regionserver.HRegion.replayWALFlushMarker(HRegion.java:4272)
- at org.apache.hadoop.hbase.regionserver.RSRpcServices.doReplayBatchOp(RSRpcServices.java:835)
- at org.apache.hadoop.hbase.regionserver.RSRpcServices.replay(RSRpcServices.java:1765)
- at org.apache.hadoop.hbase.protobuf.generated.AdminProtos$AdminService$2.callBlockingMethod(AdminProtos.java:22255)
- at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2170)
- at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:109)
- at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:133)
- at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:108)
- at java.lang.Thread.run(Thread.java:745)
- Wed Nov 09 00:43:27 UTC 2016, RpcRetryingCaller{globalStartTime=1478652206658, pause=100, retries=35}, org.apache.hadoop.hbase.regionserver.UnexpectedStateException: org.apache.hadoop.hbase.regionserver.UnexpectedStateException: Current snapshot id is -1,passed 1478639480535
- at org.apache.hadoop.hbase.regionserver.DefaultMemStore.clearSnapshot(DefaultMemStore.java:191)
- tim@staging-core-data-23d:~$ vi /mnt/xvdo/var/log/hbase/hbase-cmf-hbase-REGIONSERVER-staging-core-data-23d.log.out
- at org.apache.hadoop.hbase.regionserver.DefaultMemStore.clearSnapshot(DefaultMemStore.java:191)
- at org.apache.hadoop.hbase.regionserver.HStore.updateStorefiles(HStore.java:1082)
- at org.apache.hadoop.hbase.regionserver.HStore.access$600(HStore.java:119)
- at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.replayFlush(HStore.java:2377)
- at org.apache.hadoop.hbase.regionserver.HRegion.replayFlushInStores(HRegion.java:4565)
- at org.apache.hadoop.hbase.regionserver.HRegion.replayWALFlushCommitMarker(HRegion.java:4471)
- at org.apache.hadoop.hbase.regionserver.HRegion.replayWALFlushMarker(HRegion.java:4272)
- at org.apache.hadoop.hbase.regionserver.RSRpcServices.doReplayBatchOp(RSRpcServices.java:835)
- at org.apache.hadoop.hbase.regionserver.RSRpcServices.replay(RSRpcServices.java:1765)
- at org.apache.hadoop.hbase.protobuf.generated.AdminProtos$AdminService$2.callBlockingMethod(AdminProtos.java:22255)
- at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2170)
- at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:109)
- at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:133)
- at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:108)
- at java.lang.Thread.run(Thread.java:745)
- at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:147)
- at org.apache.hadoop.hbase.replication.regionserver.RegionReplicaReplicationEndpoint$RetryingRpcCallable.call(RegionReplicaReplicationEndpoint.java:606)
- at java.util.concurrent.FutureTask.run(FutureTask.java:266)
- at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
- at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
- at java.lang.Thread.run(Thread.java:745)
- Caused by: org.apache.hadoop.hbase.regionserver.UnexpectedStateException: org.apache.hadoop.hbase.regionserver.UnexpectedStateException: Current snapshot id is -1,passed 1478639480535
- at org.apache.hadoop.hbase.regionserver.DefaultMemStore.clearSnapshot(DefaultMemStore.java:191)
- at org.apache.hadoop.hbase.regionserver.HStore.updateStorefiles(HStore.java:1082)
- at org.apache.hadoop.hbase.regionserver.HStore.access$600(HStore.java:119)
- at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.replayFlush(HStore.java:2377)
- at org.apache.hadoop.hbase.regionserver.HRegion.replayFlushInStores(HRegion.java:4565)
- at org.apache.hadoop.hbase.regionserver.HRegion.replayWALFlushCommitMarker(HRegion.java:4471)
- at org.apache.hadoop.hbase.regionserver.HRegion.replayWALFlushMarker(HRegion.java:4272)
- at org.apache.hadoop.hbase.regionserver.RSRpcServices.doReplayBatchOp(RSRpcServices.java:835)
- at org.apache.hadoop.hbase.regionserver.RSRpcServices.replay(RSRpcServices.java:1765)
- at org.apache.hadoop.hbase.protobuf.generated.AdminProtos$AdminService$2.callBlockingMethod(AdminProtos.java:22255)
- at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2170)
- at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:109)
- at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:133)
- at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:108)
- at java.lang.Thread.run(Thread.java:745)
- at sun.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source)
- at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
- at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
- at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:106)
- at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:95)
- at org.apache.hadoop.hbase.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:327)
- at org.apache.hadoop.hbase.replication.regionserver.RegionReplicaReplicationEndpoint$RegionReplicaReplayCallable.replayToServer(RegionReplicaReplicationEndpoint.java:661)
- at org.apache.hadoop.hbase.replication.regionserver.RegionReplicaReplicationEndpoint$RegionReplicaReplayCallable.call(RegionReplicaReplicationEndpoint.java:633)
- at org.apache.hadoop.hbase.replication.regionserver.RegionReplicaReplicationEndpoint$RegionReplicaReplayCallable.call(RegionReplicaReplicationEndpoint.java:614)
- at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:126)
- ... 5 more
- Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.regionserver.UnexpectedStateException): org.apache.hadoop.hbase.regionserver.UnexpectedStateException: Current snapshot id is -1,passed 1478639480535
- at org.apache.hadoop.hbase.regionserver.DefaultMemStore.clearSnapshot(DefaultMemStore.java:191)
- at org.apache.hadoop.hbase.regionserver.HStore.updateStorefiles(HStore.java:1082)
- at org.apache.hadoop.hbase.regionserver.HStore.access$600(HStore.java:119)
- at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.replayFlush(HStore.java:2377)
- at org.apache.hadoop.hbase.regionserver.HRegion.replayFlushInStores(HRegion.java:4565)
- at org.apache.hadoop.hbase.regionserver.HRegion.replayWALFlushCommitMarker(HRegion.java:4471)
- at org.apache.hadoop.hbase.regionserver.HRegion.replayWALFlushMarker(HRegion.java:4272)
- at org.apache.hadoop.hbase.regionserver.RSRpcServices.doReplayBatchOp(RSRpcServices.java:835)
- at org.apache.hadoop.hbase.regionserver.RSRpcServices.replay(RSRpcServices.java:1765)
- at org.apache.hadoop.hbase.protobuf.generated.AdminProtos$AdminService$2.callBlockingMethod(AdminProtos.java:22255)
- at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2170)
- at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:109)
- at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:133)
- at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:108)
- at java.lang.Thread.run(Thread.java:745)
- at org.apache.hadoop.hbase.ipc.RpcClientImpl.call(RpcClientImpl.java:1268)
- at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:226)
- at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:331)
- at org.apache.hadoop.hbase.protobuf.generated.AdminProtos$AdminService$BlockingStub.replay(AdminProtos.java:23221)
- at org.apache.hadoop.hbase.replication.regionserver.RegionReplicaReplicationEndpoint$RegionReplicaReplayCallable.replayToServer(RegionReplicaReplicationEndpoint.java:659)
- ... 8 more
Advertisement
Add Comment
Please, Sign In to add comment