You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@hbase.apache.org by Erdinç Bereketoğlu <er...@gmail.com> on 2015/04/28 18:46:28 UTC
HBase standalone connection refused error
Hello,
I run HBase 1.0.0 in standalone mode for OpenTSDB. I installed and
configured it 2 weeks ago, and imported some data. But yesterday, the
service has stopped. I try to restart HBase but when I try to connect
zookeoper using telnet, I am getting "Unable to connect to remote host:
Connection refused". When I checked the HBase logs, there were a lot of
errors.
My /etc/hosts file:
```
127.0.0.1 localhost
# The following lines are desirable for IPv6 capable hosts
::1 ip6-localhost ip6-loopback
fe00::0 ip6-localnet
ff00::0 ip6-mcastprefix
ff02::1 ip6-allnodes
ff02::2 ip6-allrouters
ff02::3 ip6-allhosts
```
HBase logs:
java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at
sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:739)
at
org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:361)
at
org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
2015-04-27 21:04:14,231 WARN [main] zookeeper.RecoverableZooKeeper:
Possibly transient ZooKeeper, quorum=localhost:2181,
exception=org.apache.zookeeper.KeeperException$ConnectionLossException:
KeeperErrorCode = ConnectionLoss for /hbase
2015-04-27 21:04:14,231 ERROR [main] zookeeper.RecoverableZooKeeper:
ZooKeeper exists failed after 4 attempts
2015-04-27 21:04:14,231 WARN [main] zookeeper.ZKUtil:
hconnection-0x241cb0160x0, quorum=localhost:2181, baseZNode=/hbase Unable
to set watcher on znode (/hbase)
org.apache.zookeeper.KeeperException$ConnectionLossException:
KeeperErrorCode = ConnectionLoss for /hbase
at
org.apache.zookeeper.KeeperException.create(KeeperException.java:99)
at
org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1045)
at
org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:222)
at
org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:481)
at
org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.checkIfBaseNodeAvailable(ConnectionManager.java:849)
at
org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.access$400(ConnectionManager.java:535)
at
org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation$StubMaker.makeStubNoRetries(ConnectionManager.java:1435)
at
org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation$StubMaker.makeStub(ConnectionManager.java:1476)
at
org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation$MasterServiceStubMaker.makeStub(ConnectionManager.java:1505)
at
org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.getKeepAliveMasterService(ConnectionManager.java:1711)
at
org.apache.hadoop.hbase.client.MasterCallable.prepare(MasterCallable.java:38)
at
org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:124)
at
org.apache.hadoop.hbase.client.HBaseAdmin.executeCallable(HBaseAdmin.java:3601)
at
org.apache.hadoop.hbase.client.HBaseAdmin.shutdown(HBaseAdmin.java:2277)
at
org.apache.hadoop.hbase.master.HMasterCommandLine.stopMaster(HMasterCommandLine.java:234)
at
org.apache.hadoop.hbase.master.HMasterCommandLine.run(HMasterCommandLine.java:141)
at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:70)
at
org.apache.hadoop.hbase.util.ServerCommandLine.doMain(ServerCommandLine.java:126)
at org.apache.hadoop.hbase.master.HMaster.main(HMaster.java:1996)
2015-04-27 21:04:14,231 ERROR [main] zookeeper.ZooKeeperWatcher:
hconnection-0x241cb0160x0, quorum=localhost:2181, baseZNode=/hbase Received
unexpected KeeperException, re-throwing exception
org.apache.zookeeper.KeeperException$ConnectionLossException:
KeeperErrorCode = ConnectionLoss for /hbase
at
org.apache.zookeeper.KeeperException.create(KeeperException.java:99)
at
org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1045)
at
org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:222)
at
org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:481)
at
org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.checkIfBaseNodeAvailable(ConnectionManager.java:849)
at
org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.access$400(ConnectionManager.java:535)
at
org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation$StubMaker.makeStubNoRetries(ConnectionManager.java:1435)
at
org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation$StubMaker.makeStub(ConnectionManager.java:1476)
at
org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation$MasterServiceStubMaker.makeStub(ConnectionManager.java:1505)
at
org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.getKeepAliveMasterService(ConnectionManager.java:1711)
at
org.apache.hadoop.hbase.client.MasterCallable.prepare(MasterCallable.java:38)
at
org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:124)
at
org.apache.hadoop.hbase.client.HBaseAdmin.executeCallable(HBaseAdmin.java:3601)
at
org.apache.hadoop.hbase.client.HBaseAdmin.shutdown(HBaseAdmin.java:2277)
at
org.apache.hadoop.hbase.master.HMasterCommandLine.stopMaster(HMasterCommandLine.java:234)
at
org.apache.hadoop.hbase.master.HMasterCommandLine.run(HMasterCommandLine.java:141)
at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:70)
at
org.apache.hadoop.hbase.util.ServerCommandLine.doMain(ServerCommandLine.java:126)
at org.apache.hadoop.hbase.master.HMaster.main(HMaster.java:1996)
2015-04-27 21:04:14,232 ERROR [main]
client.ConnectionManager$HConnectionImplementation: Can't get connection to
ZooKeeper: KeeperErrorCode = ConnectionLoss for /hbase
2015-04-27 21:04:14,236 ERROR [main] master.HMasterCommandLine: Failed to
stop master
org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after
attempts=1, exceptions:
Mon Apr 27 21:04:14 UTC 2015,
RpcRetryingCaller{globalStartTime=1430168636637, pause=100, retries=1},
org.apache.hadoop.hbase.MasterNotRunningException:
org.apache.hadoop.hbase.MasterNotRunningException: Can't get connection to
ZooKeeper: KeeperErrorCode = ConnectionLoss for /hbase
at
org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:147)
at
org.apache.hadoop.hbase.client.HBaseAdmin.executeCallable(HBaseAdmin.java:3601)
at
org.apache.hadoop.hbase.client.HBaseAdmin.shutdown(HBaseAdmin.java:2277)
at
org.apache.hadoop.hbase.master.HMasterCommandLine.stopMaster(HMasterCommandLine.java:234)
at
org.apache.hadoop.hbase.master.HMasterCommandLine.run(HMasterCommandLine.java:141)
at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:70)
at
org.apache.hadoop.hbase.util.ServerCommandLine.doMain(ServerCommandLine.java:126)
at org.apache.hadoop.hbase.master.HMaster.main(HMaster.java:1996)
Caused by: org.apache.hadoop.hbase.MasterNotRunningException:
org.apache.hadoop.hbase.MasterNotRunningException: Can't get connection to
ZooKeeper: KeeperErrorCode = ConnectionLoss for /hbase
at
org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation$StubMaker.makeStub(ConnectionManager.java:1485)
at
org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation$MasterServiceStubMaker.makeStub(ConnectionManager.java:1505)
at
org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.getKeepAliveMasterService(ConnectionManager.java:1711)
at
org.apache.hadoop.hbase.client.MasterCallable.prepare(MasterCallable.java:38)
at
org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:124)
... 7 more
Caused by: org.apache.hadoop.hbase.MasterNotRunningException: Can't get
connection to ZooKeeper: KeeperErrorCode = ConnectionLoss for /hbase
at
org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.checkIfBaseNodeAvailable(ConnectionManager.java:860)
at
org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.access$400(ConnectionManager.java:535)
at
org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation$StubMaker.makeStubNoRetries(ConnectionManager.java:1435)
at
org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation$StubMaker.makeStub(ConnectionManager.java:1476)
... 11 more
Caused by: org.apache.zookeeper.KeeperException$ConnectionLossException:
KeeperErrorCode = ConnectionLoss for /hbase
at
org.apache.zookeeper.KeeperException.create(KeeperException.java:99)
at
org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1045)
at
org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:222)
at
org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:481)
at
org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.checkIfBaseNodeAvailable(ConnectionManager.java:849)
... 14 more
-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
2015-04-28 16:10:04,548 ERROR [RS_OPEN_REGION-ip-172-30-0-249:53414-1]
handler.OpenRegionHandler: Failed open of region=tsdb,\x00\x00AU
\x0B,1429656370760.9ae2dbabd3a304a60f1e976fdba1aa15., starting to roll back
the global memstore size.
org.apache.hadoop.hbase.DroppedSnapshotException: region: tsdb,\x00\x00AU
\x0B,1429656370760.9ae2dbabd3a304a60f1e976fdba1aa15.
at
org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:1999)
at
org.apache.hadoop.hbase.regionserver.HRegion.replayRecoveredEdits(HRegion.java:3692)
at
org.apache.hadoop.hbase.regionserver.HRegion.replayRecoveredEditsIfAny(HRegion.java:3497)
at
org.apache.hadoop.hbase.regionserver.HRegion.initializeRegionStores(HRegion.java:887)
at
org.apache.hadoop.hbase.regionserver.HRegion.initializeRegionInternals(HRegion.java:767)
at
org.apache.hadoop.hbase.regionserver.HRegion.initialize(HRegion.java:740)
at
org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:4908)
at
org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:4874)
at
org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:4845)
at
org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:4801)
at
org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:4752)
at
org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler.openRegion(OpenRegionHandler.java:356)
at
org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler.process(OpenRegionHandler.java:126)
at
org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:128)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.NegativeArraySizeException
at
org.apache.hadoop.hbase.CellComparator.getMinimumMidpointArray(CellComparator.java:490)
at
org.apache.hadoop.hbase.CellComparator.getMidpoint(CellComparator.java:448)
at
org.apache.hadoop.hbase.io.hfile.HFileWriterV2.finishBlock(HFileWriterV2.java:165)
at
org.apache.hadoop.hbase.io.hfile.HFileWriterV2.checkBlockBoundary(HFileWriterV2.java:146)
at
org.apache.hadoop.hbase.io.hfile.HFileWriterV2.append(HFileWriterV2.java:263)
at
org.apache.hadoop.hbase.io.hfile.HFileWriterV3.append(HFileWriterV3.java:87)
at
org.apache.hadoop.hbase.regionserver.StoreFile$Writer.append(StoreFile.java:932)
at
org.apache.hadoop.hbase.regionserver.StoreFlusher.performFlush(StoreFlusher.java:121)
at
org.apache.hadoop.hbase.regionserver.DefaultStoreFlusher.flushSnapshot(DefaultStoreFlusher.java:71)
at
org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:879)
at
org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:2128)
at
org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:1953)
... 16 more
-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
2015-04-28 16:10:29,963 INFO [main-SendThread(localhost:2181)]
zookeeper.ClientCnxn: Opening socket connection to server localhost/
127.0.0.1:2181. Will not attempt to authenticate using SASL (unknown error)
2015-04-28 16:10:29,985 WARN [main-SendThread(localhost:2181)]
zookeeper.ClientCnxn: Session 0x0 for server null, unexpected error,
closing socket connection and attempting reconnect
java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at
sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:739)
at
org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:361)
at
org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
2015-04-28 16:10:30,103 WARN [main] zookeeper.RecoverableZooKeeper:
Possibly transient ZooKeeper, quorum=localhost:2181,
exception=org.apache.zookeeper.KeeperException$ConnectionLossException:
KeeperErrorCode = ConnectionLoss for /hbase/master
2015-04-28 16:10:30,104 INFO [main] util.RetryCounter: Sleeping 1000ms
before retry #0...
2015-04-28 16:10:31,097 INFO [main-SendThread(localhost:2181)]
zookeeper.ClientCnxn: Opening socket connection to server localhost/
127.0.0.1:2181. Will not attempt to authenticate using SASL (unknown error)
2015-04-28 16:10:31,098 WARN [main-SendThread(localhost:2181)]
zookeeper.ClientCnxn: Session 0x0 for server null, unexpected error,
closing socket connection and attempting reconnect
java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at
sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:739)
at
org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:361)
at
org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
2015-04-28 16:10:31,198 WARN [main] zookeeper.RecoverableZooKeeper:
Possibly transient ZooKeeper, quorum=localhost:2181,
exception=org.apache.zookeeper.KeeperException$ConnectionLossException:
KeeperErrorCode = ConnectionLoss for /hbase/master
2015-04-28 16:10:31,198 ERROR [main] zookeeper.RecoverableZooKeeper:
ZooKeeper getData failed after 1 attempts
2015-04-28 16:10:31,198 WARN [main] zookeeper.ZKUtil: clean znode for
master0x0, quorum=localhost:2181, baseZNode=/hbase Unable to get data of
znode /hbase/master
org.apache.zookeeper.KeeperException$ConnectionLossException:
KeeperErrorCode = ConnectionLoss for /hbase/master
at
org.apache.zookeeper.KeeperException.create(KeeperException.java:99)
at
org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1155)
at
org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:360)
at
org.apache.hadoop.hbase.zookeeper.ZKUtil.getDataNoWatch(ZKUtil.java:775)
at
org.apache.hadoop.hbase.zookeeper.MasterAddressTracker.deleteIfEquals(MasterAddressTracker.java:251)
at org.apache.hadoop.hbase.ZNodeClearer.clear(ZNodeClearer.java:149)
at
org.apache.hadoop.hbase.master.HMasterCommandLine.run(HMasterCommandLine.java:143)
at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:70)
at
org.apache.hadoop.hbase.util.ServerCommandLine.doMain(ServerCommandLine.java:126)
at org.apache.hadoop.hbase.master.HMaster.main(HMaster.java:1996)
2015-04-28 16:10:31,199 ERROR [main] zookeeper.ZooKeeperWatcher: clean
znode for master0x0, quorum=localhost:2181, baseZNode=/hbase Received
unexpected KeeperException, re-throwing exception
org.apache.zookeeper.KeeperException$ConnectionLossException:
KeeperErrorCode = ConnectionLoss for /hbase/master
at
org.apache.zookeeper.KeeperException.create(KeeperException.java:99)
at
org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1155)
at
org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:360)
at
org.apache.hadoop.hbase.zookeeper.ZKUtil.getDataNoWatch(ZKUtil.java:775)
at
org.apache.hadoop.hbase.zookeeper.MasterAddressTracker.deleteIfEquals(MasterAddressTracker.java:251)
at org.apache.hadoop.hbase.ZNodeClearer.clear(ZNodeClearer.java:149)
at
org.apache.hadoop.hbase.master.HMasterCommandLine.run(HMasterCommandLine.java:143)
at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:70)
at
org.apache.hadoop.hbase.util.ServerCommandLine.doMain(ServerCommandLine.java:126)
at org.apache.hadoop.hbase.master.HMaster.main(HMaster.java:1996)
2015-04-28 16:10:31,199 WARN [main] zookeeper.ZooKeeperNodeTracker: Can't
get or delete the master znode
org.apache.zookeeper.KeeperException$ConnectionLossException:
KeeperErrorCode = ConnectionLoss for /hbase/master
at
org.apache.zookeeper.KeeperException.create(KeeperException.java:99)
at
org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1155)
at
org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:360)
at
org.apache.hadoop.hbase.zookeeper.ZKUtil.getDataNoWatch(ZKUtil.java:775)
at
org.apache.hadoop.hbase.zookeeper.MasterAddressTracker.deleteIfEquals(MasterAddressTracker.java:251)
at org.apache.hadoop.hbase.ZNodeClearer.clear(ZNodeClearer.java:149)
at
org.apache.hadoop.hbase.master.HMasterCommandLine.run(HMasterCommandLine.java:143)
at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:70)
at
org.apache.hadoop.hbase.util.ServerCommandLine.doMain(ServerCommandLine.java:126)
at org.apache.hadoop.hbase.master.HMaster.main(HMaster.java:1996)
2015-04-28 16:10:32,198 INFO [main-SendThread(localhost:2181)]
zookeeper.ClientCnxn: Opening socket connection to server localhost/
127.0.0.1:2181. Will not attempt to authenticate using SASL (unknown error)
Kind regards,
Erdinc
Re: HBase standalone connection refused error
Posted by Stack <st...@duboce.net>.
Did you alter configuration defaults (Defaults write data to /tmp dir so a
restart could cause loss if OS clears tmp).
Shutdown all HBase-related services then restart. If still does not log,
post link to complete log.
Thanks,
St.Ack
On Tue, Apr 28, 2015 at 9:46 AM, Erdinç Bereketoğlu <
erdinc.bereketoglu@gmail.com> wrote:
> Hello,
>
> I run HBase 1.0.0 in standalone mode for OpenTSDB. I installed and
> configured it 2 weeks ago, and imported some data. But yesterday, the
> service has stopped. I try to restart HBase but when I try to connect
> zookeoper using telnet, I am getting "Unable to connect to remote host:
> Connection refused". When I checked the HBase logs, there were a lot of
> errors.
>
>
> My /etc/hosts file:
>
> ```
>
> 127.0.0.1 localhost
>
> # The following lines are desirable for IPv6 capable hosts
> ::1 ip6-localhost ip6-loopback
> fe00::0 ip6-localnet
> ff00::0 ip6-mcastprefix
> ff02::1 ip6-allnodes
> ff02::2 ip6-allrouters
> ff02::3 ip6-allhosts
>
> ```
>
>
> HBase logs:
>
> java.net.ConnectException: Connection refused
> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
> at
> sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:739)
> at
>
> org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:361)
> at
> org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
> 2015-04-27 21:04:14,231 WARN [main] zookeeper.RecoverableZooKeeper:
> Possibly transient ZooKeeper, quorum=localhost:2181,
> exception=org.apache.zookeeper.KeeperException$ConnectionLossException:
> KeeperErrorCode = ConnectionLoss for /hbase
> 2015-04-27 21:04:14,231 ERROR [main] zookeeper.RecoverableZooKeeper:
> ZooKeeper exists failed after 4 attempts
> 2015-04-27 21:04:14,231 WARN [main] zookeeper.ZKUtil:
> hconnection-0x241cb0160x0, quorum=localhost:2181, baseZNode=/hbase Unable
> to set watcher on znode (/hbase)
> org.apache.zookeeper.KeeperException$ConnectionLossException:
> KeeperErrorCode = ConnectionLoss for /hbase
> at
> org.apache.zookeeper.KeeperException.create(KeeperException.java:99)
> at
> org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
> at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1045)
> at
>
> org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:222)
> at
> org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:481)
> at
>
> org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.checkIfBaseNodeAvailable(ConnectionManager.java:849)
> at
>
> org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.access$400(ConnectionManager.java:535)
> at
>
> org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation$StubMaker.makeStubNoRetries(ConnectionManager.java:1435)
> at
>
> org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation$StubMaker.makeStub(ConnectionManager.java:1476)
> at
>
> org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation$MasterServiceStubMaker.makeStub(ConnectionManager.java:1505)
> at
>
> org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.getKeepAliveMasterService(ConnectionManager.java:1711)
> at
>
> org.apache.hadoop.hbase.client.MasterCallable.prepare(MasterCallable.java:38)
> at
>
> org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:124)
> at
>
> org.apache.hadoop.hbase.client.HBaseAdmin.executeCallable(HBaseAdmin.java:3601)
> at
> org.apache.hadoop.hbase.client.HBaseAdmin.shutdown(HBaseAdmin.java:2277)
> at
>
> org.apache.hadoop.hbase.master.HMasterCommandLine.stopMaster(HMasterCommandLine.java:234)
> at
>
> org.apache.hadoop.hbase.master.HMasterCommandLine.run(HMasterCommandLine.java:141)
> at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:70)
> at
>
> org.apache.hadoop.hbase.util.ServerCommandLine.doMain(ServerCommandLine.java:126)
> at org.apache.hadoop.hbase.master.HMaster.main(HMaster.java:1996)
> 2015-04-27 21:04:14,231 ERROR [main] zookeeper.ZooKeeperWatcher:
> hconnection-0x241cb0160x0, quorum=localhost:2181, baseZNode=/hbase Received
> unexpected KeeperException, re-throwing exception
> org.apache.zookeeper.KeeperException$ConnectionLossException:
> KeeperErrorCode = ConnectionLoss for /hbase
> at
> org.apache.zookeeper.KeeperException.create(KeeperException.java:99)
> at
> org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
> at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1045)
> at
>
> org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:222)
> at
> org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:481)
> at
>
> org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.checkIfBaseNodeAvailable(ConnectionManager.java:849)
> at
>
> org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.access$400(ConnectionManager.java:535)
> at
>
> org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation$StubMaker.makeStubNoRetries(ConnectionManager.java:1435)
> at
>
> org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation$StubMaker.makeStub(ConnectionManager.java:1476)
> at
>
> org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation$MasterServiceStubMaker.makeStub(ConnectionManager.java:1505)
> at
>
> org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.getKeepAliveMasterService(ConnectionManager.java:1711)
> at
>
> org.apache.hadoop.hbase.client.MasterCallable.prepare(MasterCallable.java:38)
> at
>
> org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:124)
> at
>
> org.apache.hadoop.hbase.client.HBaseAdmin.executeCallable(HBaseAdmin.java:3601)
> at
> org.apache.hadoop.hbase.client.HBaseAdmin.shutdown(HBaseAdmin.java:2277)
> at
>
> org.apache.hadoop.hbase.master.HMasterCommandLine.stopMaster(HMasterCommandLine.java:234)
> at
>
> org.apache.hadoop.hbase.master.HMasterCommandLine.run(HMasterCommandLine.java:141)
> at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:70)
> at
>
> org.apache.hadoop.hbase.util.ServerCommandLine.doMain(ServerCommandLine.java:126)
> at org.apache.hadoop.hbase.master.HMaster.main(HMaster.java:1996)
> 2015-04-27 21:04:14,232 ERROR [main]
> client.ConnectionManager$HConnectionImplementation: Can't get connection to
> ZooKeeper: KeeperErrorCode = ConnectionLoss for /hbase
> 2015-04-27 21:04:14,236 ERROR [main] master.HMasterCommandLine: Failed to
> stop master
> org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after
> attempts=1, exceptions:
> Mon Apr 27 21:04:14 UTC 2015,
> RpcRetryingCaller{globalStartTime=1430168636637, pause=100, retries=1},
> org.apache.hadoop.hbase.MasterNotRunningException:
> org.apache.hadoop.hbase.MasterNotRunningException: Can't get connection to
> ZooKeeper: KeeperErrorCode = ConnectionLoss for /hbase
>
> at
>
> org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:147)
> at
>
> org.apache.hadoop.hbase.client.HBaseAdmin.executeCallable(HBaseAdmin.java:3601)
> at
> org.apache.hadoop.hbase.client.HBaseAdmin.shutdown(HBaseAdmin.java:2277)
> at
>
> org.apache.hadoop.hbase.master.HMasterCommandLine.stopMaster(HMasterCommandLine.java:234)
> at
>
> org.apache.hadoop.hbase.master.HMasterCommandLine.run(HMasterCommandLine.java:141)
> at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:70)
> at
>
> org.apache.hadoop.hbase.util.ServerCommandLine.doMain(ServerCommandLine.java:126)
> at org.apache.hadoop.hbase.master.HMaster.main(HMaster.java:1996)
> Caused by: org.apache.hadoop.hbase.MasterNotRunningException:
> org.apache.hadoop.hbase.MasterNotRunningException: Can't get connection to
> ZooKeeper: KeeperErrorCode = ConnectionLoss for /hbase
> at
>
> org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation$StubMaker.makeStub(ConnectionManager.java:1485)
> at
>
> org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation$MasterServiceStubMaker.makeStub(ConnectionManager.java:1505)
> at
>
> org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.getKeepAliveMasterService(ConnectionManager.java:1711)
> at
>
> org.apache.hadoop.hbase.client.MasterCallable.prepare(MasterCallable.java:38)
> at
>
> org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:124)
> ... 7 more
> Caused by: org.apache.hadoop.hbase.MasterNotRunningException: Can't get
> connection to ZooKeeper: KeeperErrorCode = ConnectionLoss for /hbase
> at
>
> org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.checkIfBaseNodeAvailable(ConnectionManager.java:860)
> at
>
> org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.access$400(ConnectionManager.java:535)
> at
>
> org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation$StubMaker.makeStubNoRetries(ConnectionManager.java:1435)
> at
>
> org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation$StubMaker.makeStub(ConnectionManager.java:1476)
> ... 11 more
> Caused by: org.apache.zookeeper.KeeperException$ConnectionLossException:
> KeeperErrorCode = ConnectionLoss for /hbase
> at
> org.apache.zookeeper.KeeperException.create(KeeperException.java:99)
> at
> org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
> at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1045)
> at
>
> org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:222)
> at
> org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:481)
> at
>
> org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.checkIfBaseNodeAvailable(ConnectionManager.java:849)
> ... 14 more
>
>
>
>
> -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
>
>
> 2015-04-28 16:10:04,548 ERROR [RS_OPEN_REGION-ip-172-30-0-249:53414-1]
> handler.OpenRegionHandler: Failed open of region=tsdb,\x00\x00AU
> \x0B,1429656370760.9ae2dbabd3a304a60f1e976fdba1aa15., starting to roll back
> the global memstore size.
> org.apache.hadoop.hbase.DroppedSnapshotException: region: tsdb,\x00\x00AU
> \x0B,1429656370760.9ae2dbabd3a304a60f1e976fdba1aa15.
> at
>
> org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:1999)
> at
>
> org.apache.hadoop.hbase.regionserver.HRegion.replayRecoveredEdits(HRegion.java:3692)
> at
>
> org.apache.hadoop.hbase.regionserver.HRegion.replayRecoveredEditsIfAny(HRegion.java:3497)
> at
>
> org.apache.hadoop.hbase.regionserver.HRegion.initializeRegionStores(HRegion.java:887)
> at
>
> org.apache.hadoop.hbase.regionserver.HRegion.initializeRegionInternals(HRegion.java:767)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.initialize(HRegion.java:740)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:4908)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:4874)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:4845)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:4801)
> at
> org.apache.hadoop.hbase.regionserver.HRegion.openHRegion(HRegion.java:4752)
> at
>
> org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler.openRegion(OpenRegionHandler.java:356)
> at
>
> org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler.process(OpenRegionHandler.java:126)
> at
> org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:128)
> at
>
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
> at
>
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
> at java.lang.Thread.run(Thread.java:745)
> Caused by: java.lang.NegativeArraySizeException
> at
>
> org.apache.hadoop.hbase.CellComparator.getMinimumMidpointArray(CellComparator.java:490)
> at
> org.apache.hadoop.hbase.CellComparator.getMidpoint(CellComparator.java:448)
> at
>
> org.apache.hadoop.hbase.io.hfile.HFileWriterV2.finishBlock(HFileWriterV2.java:165)
> at
>
> org.apache.hadoop.hbase.io.hfile.HFileWriterV2.checkBlockBoundary(HFileWriterV2.java:146)
> at
>
> org.apache.hadoop.hbase.io.hfile.HFileWriterV2.append(HFileWriterV2.java:263)
> at
>
> org.apache.hadoop.hbase.io.hfile.HFileWriterV3.append(HFileWriterV3.java:87)
> at
>
> org.apache.hadoop.hbase.regionserver.StoreFile$Writer.append(StoreFile.java:932)
> at
>
> org.apache.hadoop.hbase.regionserver.StoreFlusher.performFlush(StoreFlusher.java:121)
> at
>
> org.apache.hadoop.hbase.regionserver.DefaultStoreFlusher.flushSnapshot(DefaultStoreFlusher.java:71)
> at
> org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:879)
> at
>
> org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:2128)
> at
>
> org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:1953)
> ... 16 more
>
>
>
>
> -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
>
> 2015-04-28 16:10:29,963 INFO [main-SendThread(localhost:2181)]
> zookeeper.ClientCnxn: Opening socket connection to server localhost/
> 127.0.0.1:2181. Will not attempt to authenticate using SASL (unknown
> error)
> 2015-04-28 16:10:29,985 WARN [main-SendThread(localhost:2181)]
> zookeeper.ClientCnxn: Session 0x0 for server null, unexpected error,
> closing socket connection and attempting reconnect
> java.net.ConnectException: Connection refused
> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
> at
> sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:739)
> at
>
> org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:361)
> at
> org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
> 2015-04-28 16:10:30,103 WARN [main] zookeeper.RecoverableZooKeeper:
> Possibly transient ZooKeeper, quorum=localhost:2181,
> exception=org.apache.zookeeper.KeeperException$ConnectionLossException:
> KeeperErrorCode = ConnectionLoss for /hbase/master
> 2015-04-28 16:10:30,104 INFO [main] util.RetryCounter: Sleeping 1000ms
> before retry #0...
> 2015-04-28 16:10:31,097 INFO [main-SendThread(localhost:2181)]
> zookeeper.ClientCnxn: Opening socket connection to server localhost/
> 127.0.0.1:2181. Will not attempt to authenticate using SASL (unknown
> error)
> 2015-04-28 16:10:31,098 WARN [main-SendThread(localhost:2181)]
> zookeeper.ClientCnxn: Session 0x0 for server null, unexpected error,
> closing socket connection and attempting reconnect
> java.net.ConnectException: Connection refused
> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
> at
> sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:739)
> at
>
> org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:361)
> at
> org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
> 2015-04-28 16:10:31,198 WARN [main] zookeeper.RecoverableZooKeeper:
> Possibly transient ZooKeeper, quorum=localhost:2181,
> exception=org.apache.zookeeper.KeeperException$ConnectionLossException:
> KeeperErrorCode = ConnectionLoss for /hbase/master
> 2015-04-28 16:10:31,198 ERROR [main] zookeeper.RecoverableZooKeeper:
> ZooKeeper getData failed after 1 attempts
> 2015-04-28 16:10:31,198 WARN [main] zookeeper.ZKUtil: clean znode for
> master0x0, quorum=localhost:2181, baseZNode=/hbase Unable to get data of
> znode /hbase/master
> org.apache.zookeeper.KeeperException$ConnectionLossException:
> KeeperErrorCode = ConnectionLoss for /hbase/master
> at
> org.apache.zookeeper.KeeperException.create(KeeperException.java:99)
> at
> org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1155)
> at
>
> org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:360)
> at
> org.apache.hadoop.hbase.zookeeper.ZKUtil.getDataNoWatch(ZKUtil.java:775)
> at
>
> org.apache.hadoop.hbase.zookeeper.MasterAddressTracker.deleteIfEquals(MasterAddressTracker.java:251)
> at
> org.apache.hadoop.hbase.ZNodeClearer.clear(ZNodeClearer.java:149)
> at
>
> org.apache.hadoop.hbase.master.HMasterCommandLine.run(HMasterCommandLine.java:143)
> at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:70)
> at
>
> org.apache.hadoop.hbase.util.ServerCommandLine.doMain(ServerCommandLine.java:126)
> at org.apache.hadoop.hbase.master.HMaster.main(HMaster.java:1996)
> 2015-04-28 16:10:31,199 ERROR [main] zookeeper.ZooKeeperWatcher: clean
> znode for master0x0, quorum=localhost:2181, baseZNode=/hbase Received
> unexpected KeeperException, re-throwing exception
> org.apache.zookeeper.KeeperException$ConnectionLossException:
> KeeperErrorCode = ConnectionLoss for /hbase/master
> at
> org.apache.zookeeper.KeeperException.create(KeeperException.java:99)
> at
> org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1155)
> at
>
> org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:360)
> at
> org.apache.hadoop.hbase.zookeeper.ZKUtil.getDataNoWatch(ZKUtil.java:775)
> at
>
> org.apache.hadoop.hbase.zookeeper.MasterAddressTracker.deleteIfEquals(MasterAddressTracker.java:251)
> at
> org.apache.hadoop.hbase.ZNodeClearer.clear(ZNodeClearer.java:149)
> at
>
> org.apache.hadoop.hbase.master.HMasterCommandLine.run(HMasterCommandLine.java:143)
> at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:70)
> at
>
> org.apache.hadoop.hbase.util.ServerCommandLine.doMain(ServerCommandLine.java:126)
> at org.apache.hadoop.hbase.master.HMaster.main(HMaster.java:1996)
> 2015-04-28 16:10:31,199 WARN [main] zookeeper.ZooKeeperNodeTracker: Can't
> get or delete the master znode
> org.apache.zookeeper.KeeperException$ConnectionLossException:
> KeeperErrorCode = ConnectionLoss for /hbase/master
> at
> org.apache.zookeeper.KeeperException.create(KeeperException.java:99)
> at
> org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1155)
> at
>
> org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:360)
> at
> org.apache.hadoop.hbase.zookeeper.ZKUtil.getDataNoWatch(ZKUtil.java:775)
> at
>
> org.apache.hadoop.hbase.zookeeper.MasterAddressTracker.deleteIfEquals(MasterAddressTracker.java:251)
> at
> org.apache.hadoop.hbase.ZNodeClearer.clear(ZNodeClearer.java:149)
> at
>
> org.apache.hadoop.hbase.master.HMasterCommandLine.run(HMasterCommandLine.java:143)
> at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:70)
> at
>
> org.apache.hadoop.hbase.util.ServerCommandLine.doMain(ServerCommandLine.java:126)
> at org.apache.hadoop.hbase.master.HMaster.main(HMaster.java:1996)
> 2015-04-28 16:10:32,198 INFO [main-SendThread(localhost:2181)]
> zookeeper.ClientCnxn: Opening socket connection to server localhost/
> 127.0.0.1:2181. Will not attempt to authenticate using SASL (unknown
> error)
>
>
> Kind regards,
> Erdinc
>