You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@hbase.apache.org by Tomas Tillery <to...@gmail.com> on 2012/02/09 20:19:50 UTC

Thrift on regionservers - won't connect

Hello, all

It's my understanding that it's best practice to colocate thrift on each of
the regionservers, and use load balancing to spread queries between them.
However, I can't seem to get thrift to bind from the regionservers (only
from the hmaster).

The machines have connectivity between them; regionservers connect and
communicate with the hmaster without any problems.

I'm running Hadoop 0.20.2-cdh3u2, HBase 0.90.4-cdh3u2.

Thrift can run succesfully from the HMaster, but not from any of the
datanode/regionservers

What appears to be the relevant content from
hbase-hbase-thrift-region-1.out below:

12/02/09 19:09:17 INFO client.HConnectionManager$HConnectionImplementation:
getMaster attempt 0 of 10 failed; retrying after sleep of 1000
java.net.ConnectException: Connection refused
        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
        at
sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
        at
org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
        at
org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
        at
org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
        at
org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
        at
org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
        at $Proxy4.getProtocolVersion(Unknown Source)
        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
        at
org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
        at
org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
        at
org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
        at
org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
        at
org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
        at
org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
12/02/09 19:09:18 INFO client.HConnectionManager$HConnectionImplementation:
getMaster attempt 1 of 10 failed; retrying after sleep of 1000
java.net.ConnectException: Connection refused
        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
        at
sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
        at
org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
        at
org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
        at
org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
        at
org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
        at
org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
        at $Proxy4.getProtocolVersion(Unknown Source)
        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
        at
org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
        at
org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
        at
org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
        at
org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
        at
org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
        at
org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
12/02/09 19:09:19 INFO client.HConnectionManager$HConnectionImplementation:
getMaster attempt 2 of 10 failed; retrying after sleep of 1000
java.net.ConnectException: Connection refused
        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
        at
sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
        at
org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
        at
org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
        at
org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
        at
org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
        at
org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
        at $Proxy4.getProtocolVersion(Unknown Source)
        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
        at
org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
        at
org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
        at
org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
        at
org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
        at
org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
        at
org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
12/02/09 19:09:20 INFO client.HConnectionManager$HConnectionImplementation:
getMaster attempt 3 of 10 failed; retrying after sleep of 2000
java.net.ConnectException: Connection refused
        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
        at
sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
        at
org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
        at
org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
        at
org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
        at
org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
        at
org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
        at $Proxy4.getProtocolVersion(Unknown Source)
        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
        at
org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
        at
org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
        at
org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
        at
org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
        at
org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
        at
org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
12/02/09 19:09:22 INFO client.HConnectionManager$HConnectionImplementation:
getMaster attempt 4 of 10 failed; retrying after sleep of 2000
java.net.ConnectException: Connection refused
        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
        at
sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
        at
org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
        at
org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
        at
org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
        at
org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
        at
org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
        at $Proxy4.getProtocolVersion(Unknown Source)
        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
        at
org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
        at
org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
        at
org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
        at
org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
        at
org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
        at
org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
12/02/09 19:09:24 INFO client.HConnectionManager$HConnectionImplementation:
getMaster attempt 5 of 10 failed; retrying after sleep of 4000
java.net.ConnectException: Connection refused
        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
        at
sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
        at
org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
        at
org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
        at
org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
        at
org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
        at
org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
        at $Proxy4.getProtocolVersion(Unknown Source)
        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
        at
org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
        at
org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
        at
org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
        at
org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
        at
org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
        at
org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
12/02/09 19:09:28 INFO client.HConnectionManager$HConnectionImplementation:
getMaster attempt 6 of 10 failed; retrying after sleep of 4000
java.net.ConnectException: Connection refused
        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
        at
sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
        at
org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
        at
org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
        at
org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
        at
org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
        at
org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
        at $Proxy4.getProtocolVersion(Unknown Source)
        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
        at
org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
        at
org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
        at
org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
        at
org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
        at
org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
        at
org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
12/02/09 19:09:32 INFO client.HConnectionManager$HConnectionImplementation:
getMaster attempt 7 of 10 failed; retrying after sleep of 8000
java.net.ConnectException: Connection refused
        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
        at
sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
        at
org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
        at
org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
        at
org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
        at
org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
        at
org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
        at $Proxy4.getProtocolVersion(Unknown Source)
        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
        at
org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
        at
org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
        at
org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
        at
org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
        at
org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
        at
org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
12/02/09 19:09:40 INFO client.HConnectionManager$HConnectionImplementation:
getMaster attempt 8 of 10 failed; retrying after sleep of 16000
java.net.ConnectException: Connection refused
        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
        at
sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
        at
org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
        at
org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
        at
org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
        at
org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
        at
org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
        at $Proxy4.getProtocolVersion(Unknown Source)
        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
        at
org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
        at
org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
        at
org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
        at
org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
        at
org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
        at
org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
12/02/09 19:09:56 INFO client.HConnectionManager$HConnectionImplementation:
getMaster attempt 9 of 10 failed; no more retrying.
java.net.ConnectException: Connection refused
        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
        at
sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
        at
org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
        at
org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
        at
org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
        at
org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
        at
org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
        at $Proxy4.getProtocolVersion(Unknown Source)
        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
        at
org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
        at
org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
        at
org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
        at
org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
        at
org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
        at
org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
Exception in thread "main"
org.apache.hadoop.hbase.MasterNotRunningException: master:60000
        at
org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:396)
        at
org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
        at
org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
        at
org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
        at
org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
        at
org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)

Re: Thrift on regionservers - won't connect

Posted by Jean-Daniel Cryans <jd...@apache.org>.
So is the master alive?

On Fri, Feb 10, 2012 at 12:21 PM, Tomas Tillery <to...@gmail.com> wrote:
> I stand corrected - The region servers were still connected because they
> had remained connected. Restarting them resulted in them also receiving a
> Connection refused response. Error log follows.
>
> 12/02/10 20:12:24 INFO zookeeper.ClientCnxn: Opening socket connection to
> server master/192.168.1.1:2181
> 12/02/10 20:12:24 INFO zookeeper.ClientCnxn: Socket connection established
> to master/192.168.1.1:2181, initiating session
> 12/02/10 20:12:24 INFO zookeeper.ClientCnxn: Session establishment complete
> on server master/192.168.1.1:2181, sessionid = 0x13568dc6dfe0005,
> negotiated timeout = 40000
> 12/02/10 20:12:24 INFO ipc.HBaseRpcMetrics: Initializing RPC Metrics with
> hostName=HRegionServer, port=60020
> 12/02/10 20:12:24 INFO zookeeper.ZooKeeper: Initiating client connection,
> connectString=master:2181 sessionTimeout=180000 watcher=regionserver:60020
> 12/02/10 20:12:24 INFO zookeeper.ClientCnxn: Opening socket connection to
> server master/192.168.1.1:2181
> 12/02/10 20:12:24 INFO zookeeper.ClientCnxn: Socket connection established
> to master/192.168.1.1:2181, initiating session
> 12/02/10 20:12:24 INFO zookeeper.ClientCnxn: Session establishment complete
> on server master/192.168.1.1:2181, sessionid = 0x13568dc6dfe000c,
> negotiated timeout = 40000
> 12/02/10 20:12:24 INFO regionserver.ShutdownHook: Installed shutdown hook
> thread: Shutdownhook:regionserver60020
> 12/02/10 20:12:24 INFO regionserver.MemStoreFlusher:
> globalMemStoreLimit=395.0m, globalMemStoreLimitLowMark=345.6m,
> maxHeap=987.5m
> 12/02/10 20:12:24 INFO regionserver.HRegionServer: Runs every 10000000ms
> 12/02/10 20:12:24 INFO regionserver.HRegionServer: Attempting connect to
> Master server at master:60000
> 12/02/10 20:13:24 WARN regionserver.HRegionServer: Unable to connect to
> master. Retrying. Error was:
> java.net.ConnectException: Connection refused
>        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>        at
> sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
>        at
> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
>        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
>        at
> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
>        at
> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
>        at
> org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
>        at
> org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
>        at $Proxy6.getProtocolVersion(Unknown Source)
>        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
>        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
>        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
>        at
> org.apache.hadoop.hbase.ipc.HBaseRPC.waitForProxy(HBaseRPC.java:349)
>        at
> org.apache.hadoop.hbase.regionserver.HRegionServer.getMaster(HRegionServer.java:1477)
>        at
> org.apache.hadoop.hbase.regionserver.HRegionServer.reportForDuty(HRegionServer.java:1530)
>        at
> org.apache.hadoop.hbase.regionserver.HRegionServer.tryReportForDuty(HRegionServer.java:1514)
>        at
> org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:573)
>        at java.lang.Thread.run(Thread.java:619)
>
> On Thu, Feb 9, 2012 at 2:48 PM, Tomas Tillery <to...@gmail.com>wrote:
>
>> That appears to be the case. Here's an excerpt from the regionserver log
>> on the same machine:
>>
>> 12/01/24 22:11:44 INFO regionserver.HRegionServer: Attempting connect to
>> Master server at master:60000
>> 12/01/24 22:11:44 INFO regionserver.HRegionServer: Connected to master at
>> master:60000
>> 12/01/24 22:11:44 INFO regionserver.HRegionServer: Telling master at
>> master:60000 that we are up
>> 12/01/24 22:11:44 INFO regionserver.HRegionServer: Master passed us
>> address to use. Was=region-1:60020, Now=region-1:60020
>>
>>
>> On Thu, Feb 9, 2012 at 2:43 PM, Jean-Daniel Cryans <jd...@apache.org>wrote:
>>
>>> 60010 is the web ui port by default, 60000 is the port the master uses
>>> for communication. Clients (like thrift) and region servers don't rely
>>> on a configured port because the master already posts it in zookeeper
>>> along with its address (that's how your thrift server found it).
>>>
>>> If your region servers are using master:60000 and they don't have any
>>> problem connecting to a port that doesn't respond, we have a bigger
>>> issue here :)
>>>
>>> J-D
>>>
>>> On Thu, Feb 9, 2012 at 11:38 AM, Tomas Tillery <to...@gmail.com>
>>> wrote:
>>> > No, it isn't - as far as I know, it should be aiming for 60010, rather
>>> than
>>> > 60000 (though HBase is also bound to 60000, there's no telnet response
>>> > there). There's probably something obvious I'm not managing to see, but
>>> I
>>> > can't find a way to redirect it to the proper port.
>>> > Starting thrift with this install is a matter of invoking
>>> > /etc/init.d/hadoop-hbase-thrift. Since it isn't invoked through the
>>> shell
>>> > script, I'm not sure where to specify the port to connect to.
>>> >
>>> > On Thu, Feb 9, 2012 at 2:27 PM, Jean-Daniel Cryans <jdcryans@apache.org
>>> >wrote:
>>> >
>>> >> Connection refused is a java exception that usually means that either
>>> >> there's nothing listening on the port or it's firewall'd. Is
>>> >> "master:60000" really what you're expecting to see here? Should be
>>> >> easy to "telnet master 60000" from that machien to really verify that
>>> >> it works as expected.
>>> >>
>>> >> J-D
>>> >>
>>> >> On Thu, Feb 9, 2012 at 11:19 AM, Tomas Tillery <tomastillery@gmail.com
>>> >
>>> >> wrote:
>>> >> > Hello, all
>>> >> >
>>> >> > It's my understanding that it's best practice to colocate thrift on
>>> each
>>> >> of
>>> >> > the regionservers, and use load balancing to spread queries between
>>> them.
>>> >> > However, I can't seem to get thrift to bind from the regionservers
>>> (only
>>> >> > from the hmaster).
>>> >> >
>>> >> > The machines have connectivity between them; regionservers connect
>>> and
>>> >> > communicate with the hmaster without any problems.
>>> >> >
>>> >> > I'm running Hadoop 0.20.2-cdh3u2, HBase 0.90.4-cdh3u2.
>>> >> >
>>> >> > Thrift can run succesfully from the HMaster, but not from any of the
>>> >> > datanode/regionservers
>>> >> >
>>> >> > What appears to be the relevant content from
>>> >> > hbase-hbase-thrift-region-1.out below:
>>> >> >
>>> >> > 12/02/09 19:09:17 INFO
>>> >> client.HConnectionManager$HConnectionImplementation:
>>> >> > getMaster attempt 0 of 10 failed; retrying after sleep of 1000
>>> >> > java.net.ConnectException: Connection refused
>>> >> >        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>>> >> >        at
>>> >> >
>>> sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
>>> >> >        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
>>> >> >        at
>>> >> > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
>>> >> >        at
>>> >> >
>>> org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
>>> >> >        at $Proxy4.getProtocolVersion(Unknown Source)
>>> >> >        at
>>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
>>> >> >        at
>>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
>>> >> >        at
>>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
>>> >> >        at
>>> >> > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
>>> >> >        at
>>> >> >
>>> org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
>>> >> >        at
>>> >> >
>>> org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
>>> >> > 12/02/09 19:09:18 INFO
>>> >> client.HConnectionManager$HConnectionImplementation:
>>> >> > getMaster attempt 1 of 10 failed; retrying after sleep of 1000
>>> >> > java.net.ConnectException: Connection refused
>>> >> >        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>>> >> >        at
>>> >> >
>>> sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
>>> >> >        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
>>> >> >        at
>>> >> > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
>>> >> >        at
>>> >> >
>>> org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
>>> >> >        at $Proxy4.getProtocolVersion(Unknown Source)
>>> >> >        at
>>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
>>> >> >        at
>>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
>>> >> >        at
>>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
>>> >> >        at
>>> >> > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
>>> >> >        at
>>> >> >
>>> org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
>>> >> >        at
>>> >> >
>>> org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
>>> >> > 12/02/09 19:09:19 INFO
>>> >> client.HConnectionManager$HConnectionImplementation:
>>> >> > getMaster attempt 2 of 10 failed; retrying after sleep of 1000
>>> >> > java.net.ConnectException: Connection refused
>>> >> >        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>>> >> >        at
>>> >> >
>>> sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
>>> >> >        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
>>> >> >        at
>>> >> > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
>>> >> >        at
>>> >> >
>>> org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
>>> >> >        at $Proxy4.getProtocolVersion(Unknown Source)
>>> >> >        at
>>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
>>> >> >        at
>>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
>>> >> >        at
>>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
>>> >> >        at
>>> >> > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
>>> >> >        at
>>> >> >
>>> org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
>>> >> >        at
>>> >> >
>>> org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
>>> >> > 12/02/09 19:09:20 INFO
>>> >> client.HConnectionManager$HConnectionImplementation:
>>> >> > getMaster attempt 3 of 10 failed; retrying after sleep of 2000
>>> >> > java.net.ConnectException: Connection refused
>>> >> >        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>>> >> >        at
>>> >> >
>>> sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
>>> >> >        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
>>> >> >        at
>>> >> > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
>>> >> >        at
>>> >> >
>>> org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
>>> >> >        at $Proxy4.getProtocolVersion(Unknown Source)
>>> >> >        at
>>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
>>> >> >        at
>>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
>>> >> >        at
>>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
>>> >> >        at
>>> >> > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
>>> >> >        at
>>> >> >
>>> org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
>>> >> >        at
>>> >> >
>>> org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
>>> >> > 12/02/09 19:09:22 INFO
>>> >> client.HConnectionManager$HConnectionImplementation:
>>> >> > getMaster attempt 4 of 10 failed; retrying after sleep of 2000
>>> >> > java.net.ConnectException: Connection refused
>>> >> >        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>>> >> >        at
>>> >> >
>>> sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
>>> >> >        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
>>> >> >        at
>>> >> > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
>>> >> >        at
>>> >> >
>>> org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
>>> >> >        at $Proxy4.getProtocolVersion(Unknown Source)
>>> >> >        at
>>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
>>> >> >        at
>>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
>>> >> >        at
>>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
>>> >> >        at
>>> >> > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
>>> >> >        at
>>> >> >
>>> org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
>>> >> >        at
>>> >> >
>>> org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
>>> >> > 12/02/09 19:09:24 INFO
>>> >> client.HConnectionManager$HConnectionImplementation:
>>> >> > getMaster attempt 5 of 10 failed; retrying after sleep of 4000
>>> >> > java.net.ConnectException: Connection refused
>>> >> >        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>>> >> >        at
>>> >> >
>>> sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
>>> >> >        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
>>> >> >        at
>>> >> > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
>>> >> >        at
>>> >> >
>>> org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
>>> >> >        at $Proxy4.getProtocolVersion(Unknown Source)
>>> >> >        at
>>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
>>> >> >        at
>>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
>>> >> >        at
>>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
>>> >> >        at
>>> >> > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
>>> >> >        at
>>> >> >
>>> org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
>>> >> >        at
>>> >> >
>>> org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
>>> >> > 12/02/09 19:09:28 INFO
>>> >> client.HConnectionManager$HConnectionImplementation:
>>> >> > getMaster attempt 6 of 10 failed; retrying after sleep of 4000
>>> >> > java.net.ConnectException: Connection refused
>>> >> >        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>>> >> >        at
>>> >> >
>>> sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
>>> >> >        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
>>> >> >        at
>>> >> > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
>>> >> >        at
>>> >> >
>>> org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
>>> >> >        at $Proxy4.getProtocolVersion(Unknown Source)
>>> >> >        at
>>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
>>> >> >        at
>>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
>>> >> >        at
>>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
>>> >> >        at
>>> >> > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
>>> >> >        at
>>> >> >
>>> org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
>>> >> >        at
>>> >> >
>>> org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
>>> >> > 12/02/09 19:09:32 INFO
>>> >> client.HConnectionManager$HConnectionImplementation:
>>> >> > getMaster attempt 7 of 10 failed; retrying after sleep of 8000
>>> >> > java.net.ConnectException: Connection refused
>>> >> >        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>>> >> >        at
>>> >> >
>>> sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
>>> >> >        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
>>> >> >        at
>>> >> > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
>>> >> >        at
>>> >> >
>>> org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
>>> >> >        at $Proxy4.getProtocolVersion(Unknown Source)
>>> >> >        at
>>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
>>> >> >        at
>>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
>>> >> >        at
>>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
>>> >> >        at
>>> >> > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
>>> >> >        at
>>> >> >
>>> org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
>>> >> >        at
>>> >> >
>>> org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
>>> >> > 12/02/09 19:09:40 INFO
>>> >> client.HConnectionManager$HConnectionImplementation:
>>> >> > getMaster attempt 8 of 10 failed; retrying after sleep of 16000
>>> >> > java.net.ConnectException: Connection refused
>>> >> >        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>>> >> >        at
>>> >> >
>>> sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
>>> >> >        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
>>> >> >        at
>>> >> > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
>>> >> >        at
>>> >> >
>>> org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
>>> >> >        at $Proxy4.getProtocolVersion(Unknown Source)
>>> >> >        at
>>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
>>> >> >        at
>>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
>>> >> >        at
>>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
>>> >> >        at
>>> >> > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
>>> >> >        at
>>> >> >
>>> org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
>>> >> >        at
>>> >> >
>>> org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
>>> >> > 12/02/09 19:09:56 INFO
>>> >> client.HConnectionManager$HConnectionImplementation:
>>> >> > getMaster attempt 9 of 10 failed; no more retrying.
>>> >> > java.net.ConnectException: Connection refused
>>> >> >        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>>> >> >        at
>>> >> >
>>> sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
>>> >> >        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
>>> >> >        at
>>> >> > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
>>> >> >        at
>>> >> >
>>> org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
>>> >> >        at $Proxy4.getProtocolVersion(Unknown Source)
>>> >> >        at
>>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
>>> >> >        at
>>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
>>> >> >        at
>>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
>>> >> >        at
>>> >> > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
>>> >> >        at
>>> >> >
>>> org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
>>> >> >        at
>>> >> >
>>> org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
>>> >> > Exception in thread "main"
>>> >> > org.apache.hadoop.hbase.MasterNotRunningException: master:60000
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:396)
>>> >> >        at
>>> >> > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
>>> >> >        at
>>> >> >
>>> >>
>>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
>>> >> >        at
>>> >> >
>>> org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
>>> >> >        at
>>> >> >
>>> org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
>>> >>
>>>
>>
>>

Re: Thrift on regionservers - won't connect

Posted by Tomas Tillery <to...@gmail.com>.
I stand corrected - The region servers were still connected because they
had remained connected. Restarting them resulted in them also receiving a
Connection refused response. Error log follows.

12/02/10 20:12:24 INFO zookeeper.ClientCnxn: Opening socket connection to
server master/192.168.1.1:2181
12/02/10 20:12:24 INFO zookeeper.ClientCnxn: Socket connection established
to master/192.168.1.1:2181, initiating session
12/02/10 20:12:24 INFO zookeeper.ClientCnxn: Session establishment complete
on server master/192.168.1.1:2181, sessionid = 0x13568dc6dfe0005,
negotiated timeout = 40000
12/02/10 20:12:24 INFO ipc.HBaseRpcMetrics: Initializing RPC Metrics with
hostName=HRegionServer, port=60020
12/02/10 20:12:24 INFO zookeeper.ZooKeeper: Initiating client connection,
connectString=master:2181 sessionTimeout=180000 watcher=regionserver:60020
12/02/10 20:12:24 INFO zookeeper.ClientCnxn: Opening socket connection to
server master/192.168.1.1:2181
12/02/10 20:12:24 INFO zookeeper.ClientCnxn: Socket connection established
to master/192.168.1.1:2181, initiating session
12/02/10 20:12:24 INFO zookeeper.ClientCnxn: Session establishment complete
on server master/192.168.1.1:2181, sessionid = 0x13568dc6dfe000c,
negotiated timeout = 40000
12/02/10 20:12:24 INFO regionserver.ShutdownHook: Installed shutdown hook
thread: Shutdownhook:regionserver60020
12/02/10 20:12:24 INFO regionserver.MemStoreFlusher:
globalMemStoreLimit=395.0m, globalMemStoreLimitLowMark=345.6m,
maxHeap=987.5m
12/02/10 20:12:24 INFO regionserver.HRegionServer: Runs every 10000000ms
12/02/10 20:12:24 INFO regionserver.HRegionServer: Attempting connect to
Master server at master:60000
12/02/10 20:13:24 WARN regionserver.HRegionServer: Unable to connect to
master. Retrying. Error was:
java.net.ConnectException: Connection refused
        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
        at
sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
        at
org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
        at
org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
        at
org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
        at
org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
        at
org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
        at $Proxy6.getProtocolVersion(Unknown Source)
        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
        at
org.apache.hadoop.hbase.ipc.HBaseRPC.waitForProxy(HBaseRPC.java:349)
        at
org.apache.hadoop.hbase.regionserver.HRegionServer.getMaster(HRegionServer.java:1477)
        at
org.apache.hadoop.hbase.regionserver.HRegionServer.reportForDuty(HRegionServer.java:1530)
        at
org.apache.hadoop.hbase.regionserver.HRegionServer.tryReportForDuty(HRegionServer.java:1514)
        at
org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:573)
        at java.lang.Thread.run(Thread.java:619)

On Thu, Feb 9, 2012 at 2:48 PM, Tomas Tillery <to...@gmail.com>wrote:

> That appears to be the case. Here's an excerpt from the regionserver log
> on the same machine:
>
> 12/01/24 22:11:44 INFO regionserver.HRegionServer: Attempting connect to
> Master server at master:60000
> 12/01/24 22:11:44 INFO regionserver.HRegionServer: Connected to master at
> master:60000
> 12/01/24 22:11:44 INFO regionserver.HRegionServer: Telling master at
> master:60000 that we are up
> 12/01/24 22:11:44 INFO regionserver.HRegionServer: Master passed us
> address to use. Was=region-1:60020, Now=region-1:60020
>
>
> On Thu, Feb 9, 2012 at 2:43 PM, Jean-Daniel Cryans <jd...@apache.org>wrote:
>
>> 60010 is the web ui port by default, 60000 is the port the master uses
>> for communication. Clients (like thrift) and region servers don't rely
>> on a configured port because the master already posts it in zookeeper
>> along with its address (that's how your thrift server found it).
>>
>> If your region servers are using master:60000 and they don't have any
>> problem connecting to a port that doesn't respond, we have a bigger
>> issue here :)
>>
>> J-D
>>
>> On Thu, Feb 9, 2012 at 11:38 AM, Tomas Tillery <to...@gmail.com>
>> wrote:
>> > No, it isn't - as far as I know, it should be aiming for 60010, rather
>> than
>> > 60000 (though HBase is also bound to 60000, there's no telnet response
>> > there). There's probably something obvious I'm not managing to see, but
>> I
>> > can't find a way to redirect it to the proper port.
>> > Starting thrift with this install is a matter of invoking
>> > /etc/init.d/hadoop-hbase-thrift. Since it isn't invoked through the
>> shell
>> > script, I'm not sure where to specify the port to connect to.
>> >
>> > On Thu, Feb 9, 2012 at 2:27 PM, Jean-Daniel Cryans <jdcryans@apache.org
>> >wrote:
>> >
>> >> Connection refused is a java exception that usually means that either
>> >> there's nothing listening on the port or it's firewall'd. Is
>> >> "master:60000" really what you're expecting to see here? Should be
>> >> easy to "telnet master 60000" from that machien to really verify that
>> >> it works as expected.
>> >>
>> >> J-D
>> >>
>> >> On Thu, Feb 9, 2012 at 11:19 AM, Tomas Tillery <tomastillery@gmail.com
>> >
>> >> wrote:
>> >> > Hello, all
>> >> >
>> >> > It's my understanding that it's best practice to colocate thrift on
>> each
>> >> of
>> >> > the regionservers, and use load balancing to spread queries between
>> them.
>> >> > However, I can't seem to get thrift to bind from the regionservers
>> (only
>> >> > from the hmaster).
>> >> >
>> >> > The machines have connectivity between them; regionservers connect
>> and
>> >> > communicate with the hmaster without any problems.
>> >> >
>> >> > I'm running Hadoop 0.20.2-cdh3u2, HBase 0.90.4-cdh3u2.
>> >> >
>> >> > Thrift can run succesfully from the HMaster, but not from any of the
>> >> > datanode/regionservers
>> >> >
>> >> > What appears to be the relevant content from
>> >> > hbase-hbase-thrift-region-1.out below:
>> >> >
>> >> > 12/02/09 19:09:17 INFO
>> >> client.HConnectionManager$HConnectionImplementation:
>> >> > getMaster attempt 0 of 10 failed; retrying after sleep of 1000
>> >> > java.net.ConnectException: Connection refused
>> >> >        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>> >> >        at
>> >> >
>> sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
>> >> >        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
>> >> >        at
>> >> > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
>> >> >        at
>> >> >
>> org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
>> >> >        at $Proxy4.getProtocolVersion(Unknown Source)
>> >> >        at
>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
>> >> >        at
>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
>> >> >        at
>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
>> >> >        at
>> >> > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
>> >> >        at
>> >> >
>> org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
>> >> >        at
>> >> >
>> org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
>> >> > 12/02/09 19:09:18 INFO
>> >> client.HConnectionManager$HConnectionImplementation:
>> >> > getMaster attempt 1 of 10 failed; retrying after sleep of 1000
>> >> > java.net.ConnectException: Connection refused
>> >> >        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>> >> >        at
>> >> >
>> sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
>> >> >        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
>> >> >        at
>> >> > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
>> >> >        at
>> >> >
>> org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
>> >> >        at $Proxy4.getProtocolVersion(Unknown Source)
>> >> >        at
>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
>> >> >        at
>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
>> >> >        at
>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
>> >> >        at
>> >> > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
>> >> >        at
>> >> >
>> org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
>> >> >        at
>> >> >
>> org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
>> >> > 12/02/09 19:09:19 INFO
>> >> client.HConnectionManager$HConnectionImplementation:
>> >> > getMaster attempt 2 of 10 failed; retrying after sleep of 1000
>> >> > java.net.ConnectException: Connection refused
>> >> >        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>> >> >        at
>> >> >
>> sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
>> >> >        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
>> >> >        at
>> >> > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
>> >> >        at
>> >> >
>> org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
>> >> >        at $Proxy4.getProtocolVersion(Unknown Source)
>> >> >        at
>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
>> >> >        at
>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
>> >> >        at
>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
>> >> >        at
>> >> > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
>> >> >        at
>> >> >
>> org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
>> >> >        at
>> >> >
>> org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
>> >> > 12/02/09 19:09:20 INFO
>> >> client.HConnectionManager$HConnectionImplementation:
>> >> > getMaster attempt 3 of 10 failed; retrying after sleep of 2000
>> >> > java.net.ConnectException: Connection refused
>> >> >        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>> >> >        at
>> >> >
>> sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
>> >> >        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
>> >> >        at
>> >> > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
>> >> >        at
>> >> >
>> org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
>> >> >        at $Proxy4.getProtocolVersion(Unknown Source)
>> >> >        at
>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
>> >> >        at
>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
>> >> >        at
>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
>> >> >        at
>> >> > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
>> >> >        at
>> >> >
>> org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
>> >> >        at
>> >> >
>> org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
>> >> > 12/02/09 19:09:22 INFO
>> >> client.HConnectionManager$HConnectionImplementation:
>> >> > getMaster attempt 4 of 10 failed; retrying after sleep of 2000
>> >> > java.net.ConnectException: Connection refused
>> >> >        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>> >> >        at
>> >> >
>> sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
>> >> >        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
>> >> >        at
>> >> > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
>> >> >        at
>> >> >
>> org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
>> >> >        at $Proxy4.getProtocolVersion(Unknown Source)
>> >> >        at
>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
>> >> >        at
>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
>> >> >        at
>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
>> >> >        at
>> >> > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
>> >> >        at
>> >> >
>> org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
>> >> >        at
>> >> >
>> org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
>> >> > 12/02/09 19:09:24 INFO
>> >> client.HConnectionManager$HConnectionImplementation:
>> >> > getMaster attempt 5 of 10 failed; retrying after sleep of 4000
>> >> > java.net.ConnectException: Connection refused
>> >> >        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>> >> >        at
>> >> >
>> sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
>> >> >        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
>> >> >        at
>> >> > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
>> >> >        at
>> >> >
>> org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
>> >> >        at $Proxy4.getProtocolVersion(Unknown Source)
>> >> >        at
>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
>> >> >        at
>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
>> >> >        at
>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
>> >> >        at
>> >> > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
>> >> >        at
>> >> >
>> org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
>> >> >        at
>> >> >
>> org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
>> >> > 12/02/09 19:09:28 INFO
>> >> client.HConnectionManager$HConnectionImplementation:
>> >> > getMaster attempt 6 of 10 failed; retrying after sleep of 4000
>> >> > java.net.ConnectException: Connection refused
>> >> >        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>> >> >        at
>> >> >
>> sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
>> >> >        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
>> >> >        at
>> >> > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
>> >> >        at
>> >> >
>> org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
>> >> >        at $Proxy4.getProtocolVersion(Unknown Source)
>> >> >        at
>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
>> >> >        at
>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
>> >> >        at
>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
>> >> >        at
>> >> > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
>> >> >        at
>> >> >
>> org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
>> >> >        at
>> >> >
>> org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
>> >> > 12/02/09 19:09:32 INFO
>> >> client.HConnectionManager$HConnectionImplementation:
>> >> > getMaster attempt 7 of 10 failed; retrying after sleep of 8000
>> >> > java.net.ConnectException: Connection refused
>> >> >        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>> >> >        at
>> >> >
>> sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
>> >> >        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
>> >> >        at
>> >> > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
>> >> >        at
>> >> >
>> org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
>> >> >        at $Proxy4.getProtocolVersion(Unknown Source)
>> >> >        at
>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
>> >> >        at
>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
>> >> >        at
>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
>> >> >        at
>> >> > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
>> >> >        at
>> >> >
>> org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
>> >> >        at
>> >> >
>> org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
>> >> > 12/02/09 19:09:40 INFO
>> >> client.HConnectionManager$HConnectionImplementation:
>> >> > getMaster attempt 8 of 10 failed; retrying after sleep of 16000
>> >> > java.net.ConnectException: Connection refused
>> >> >        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>> >> >        at
>> >> >
>> sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
>> >> >        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
>> >> >        at
>> >> > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
>> >> >        at
>> >> >
>> org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
>> >> >        at $Proxy4.getProtocolVersion(Unknown Source)
>> >> >        at
>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
>> >> >        at
>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
>> >> >        at
>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
>> >> >        at
>> >> > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
>> >> >        at
>> >> >
>> org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
>> >> >        at
>> >> >
>> org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
>> >> > 12/02/09 19:09:56 INFO
>> >> client.HConnectionManager$HConnectionImplementation:
>> >> > getMaster attempt 9 of 10 failed; no more retrying.
>> >> > java.net.ConnectException: Connection refused
>> >> >        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>> >> >        at
>> >> >
>> sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
>> >> >        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
>> >> >        at
>> >> > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
>> >> >        at
>> >> >
>> org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
>> >> >        at $Proxy4.getProtocolVersion(Unknown Source)
>> >> >        at
>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
>> >> >        at
>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
>> >> >        at
>> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
>> >> >        at
>> >> > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
>> >> >        at
>> >> >
>> org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
>> >> >        at
>> >> >
>> org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
>> >> > Exception in thread "main"
>> >> > org.apache.hadoop.hbase.MasterNotRunningException: master:60000
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:396)
>> >> >        at
>> >> > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
>> >> >        at
>> >> >
>> >>
>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
>> >> >        at
>> >> >
>> org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
>> >> >        at
>> >> >
>> org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
>> >>
>>
>
>

Re: Thrift on regionservers - won't connect

Posted by Tomas Tillery <to...@gmail.com>.
That appears to be the case. Here's an excerpt from the regionserver log on
the same machine:

12/01/24 22:11:44 INFO regionserver.HRegionServer: Attempting connect to
Master server at master:60000
12/01/24 22:11:44 INFO regionserver.HRegionServer: Connected to master at
master:60000
12/01/24 22:11:44 INFO regionserver.HRegionServer: Telling master at
master:60000 that we are up
12/01/24 22:11:44 INFO regionserver.HRegionServer: Master passed us address
to use. Was=region-1:60020, Now=region-1:60020

On Thu, Feb 9, 2012 at 2:43 PM, Jean-Daniel Cryans <jd...@apache.org>wrote:

> 60010 is the web ui port by default, 60000 is the port the master uses
> for communication. Clients (like thrift) and region servers don't rely
> on a configured port because the master already posts it in zookeeper
> along with its address (that's how your thrift server found it).
>
> If your region servers are using master:60000 and they don't have any
> problem connecting to a port that doesn't respond, we have a bigger
> issue here :)
>
> J-D
>
> On Thu, Feb 9, 2012 at 11:38 AM, Tomas Tillery <to...@gmail.com>
> wrote:
> > No, it isn't - as far as I know, it should be aiming for 60010, rather
> than
> > 60000 (though HBase is also bound to 60000, there's no telnet response
> > there). There's probably something obvious I'm not managing to see, but I
> > can't find a way to redirect it to the proper port.
> > Starting thrift with this install is a matter of invoking
> > /etc/init.d/hadoop-hbase-thrift. Since it isn't invoked through the shell
> > script, I'm not sure where to specify the port to connect to.
> >
> > On Thu, Feb 9, 2012 at 2:27 PM, Jean-Daniel Cryans <jdcryans@apache.org
> >wrote:
> >
> >> Connection refused is a java exception that usually means that either
> >> there's nothing listening on the port or it's firewall'd. Is
> >> "master:60000" really what you're expecting to see here? Should be
> >> easy to "telnet master 60000" from that machien to really verify that
> >> it works as expected.
> >>
> >> J-D
> >>
> >> On Thu, Feb 9, 2012 at 11:19 AM, Tomas Tillery <to...@gmail.com>
> >> wrote:
> >> > Hello, all
> >> >
> >> > It's my understanding that it's best practice to colocate thrift on
> each
> >> of
> >> > the regionservers, and use load balancing to spread queries between
> them.
> >> > However, I can't seem to get thrift to bind from the regionservers
> (only
> >> > from the hmaster).
> >> >
> >> > The machines have connectivity between them; regionservers connect and
> >> > communicate with the hmaster without any problems.
> >> >
> >> > I'm running Hadoop 0.20.2-cdh3u2, HBase 0.90.4-cdh3u2.
> >> >
> >> > Thrift can run succesfully from the HMaster, but not from any of the
> >> > datanode/regionservers
> >> >
> >> > What appears to be the relevant content from
> >> > hbase-hbase-thrift-region-1.out below:
> >> >
> >> > 12/02/09 19:09:17 INFO
> >> client.HConnectionManager$HConnectionImplementation:
> >> > getMaster attempt 0 of 10 failed; retrying after sleep of 1000
> >> > java.net.ConnectException: Connection refused
> >> >        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
> >> >        at
> >> > sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
> >> >        at
> >> >
> >>
> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
> >> >        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
> >> >        at
> >> >
> >>
> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
> >> >        at
> >> >
> >>
> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
> >> >        at
> >> > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
> >> >        at
> >> > org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
> >> >        at $Proxy4.getProtocolVersion(Unknown Source)
> >> >        at
> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
> >> >        at
> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
> >> >        at
> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
> >> >        at
> >> >
> >>
> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
> >> >        at
> >> > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
> >> >        at
> >> >
> >>
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
> >> >        at
> >> >
> >>
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
> >> >        at
> >> >
> org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
> >> >        at
> >> >
> org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
> >> > 12/02/09 19:09:18 INFO
> >> client.HConnectionManager$HConnectionImplementation:
> >> > getMaster attempt 1 of 10 failed; retrying after sleep of 1000
> >> > java.net.ConnectException: Connection refused
> >> >        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
> >> >        at
> >> > sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
> >> >        at
> >> >
> >>
> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
> >> >        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
> >> >        at
> >> >
> >>
> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
> >> >        at
> >> >
> >>
> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
> >> >        at
> >> > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
> >> >        at
> >> > org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
> >> >        at $Proxy4.getProtocolVersion(Unknown Source)
> >> >        at
> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
> >> >        at
> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
> >> >        at
> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
> >> >        at
> >> >
> >>
> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
> >> >        at
> >> > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
> >> >        at
> >> >
> >>
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
> >> >        at
> >> >
> >>
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
> >> >        at
> >> >
> org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
> >> >        at
> >> >
> org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
> >> > 12/02/09 19:09:19 INFO
> >> client.HConnectionManager$HConnectionImplementation:
> >> > getMaster attempt 2 of 10 failed; retrying after sleep of 1000
> >> > java.net.ConnectException: Connection refused
> >> >        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
> >> >        at
> >> > sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
> >> >        at
> >> >
> >>
> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
> >> >        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
> >> >        at
> >> >
> >>
> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
> >> >        at
> >> >
> >>
> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
> >> >        at
> >> > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
> >> >        at
> >> > org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
> >> >        at $Proxy4.getProtocolVersion(Unknown Source)
> >> >        at
> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
> >> >        at
> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
> >> >        at
> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
> >> >        at
> >> >
> >>
> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
> >> >        at
> >> > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
> >> >        at
> >> >
> >>
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
> >> >        at
> >> >
> >>
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
> >> >        at
> >> >
> org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
> >> >        at
> >> >
> org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
> >> > 12/02/09 19:09:20 INFO
> >> client.HConnectionManager$HConnectionImplementation:
> >> > getMaster attempt 3 of 10 failed; retrying after sleep of 2000
> >> > java.net.ConnectException: Connection refused
> >> >        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
> >> >        at
> >> > sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
> >> >        at
> >> >
> >>
> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
> >> >        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
> >> >        at
> >> >
> >>
> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
> >> >        at
> >> >
> >>
> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
> >> >        at
> >> > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
> >> >        at
> >> > org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
> >> >        at $Proxy4.getProtocolVersion(Unknown Source)
> >> >        at
> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
> >> >        at
> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
> >> >        at
> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
> >> >        at
> >> >
> >>
> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
> >> >        at
> >> > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
> >> >        at
> >> >
> >>
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
> >> >        at
> >> >
> >>
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
> >> >        at
> >> >
> org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
> >> >        at
> >> >
> org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
> >> > 12/02/09 19:09:22 INFO
> >> client.HConnectionManager$HConnectionImplementation:
> >> > getMaster attempt 4 of 10 failed; retrying after sleep of 2000
> >> > java.net.ConnectException: Connection refused
> >> >        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
> >> >        at
> >> > sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
> >> >        at
> >> >
> >>
> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
> >> >        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
> >> >        at
> >> >
> >>
> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
> >> >        at
> >> >
> >>
> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
> >> >        at
> >> > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
> >> >        at
> >> > org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
> >> >        at $Proxy4.getProtocolVersion(Unknown Source)
> >> >        at
> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
> >> >        at
> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
> >> >        at
> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
> >> >        at
> >> >
> >>
> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
> >> >        at
> >> > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
> >> >        at
> >> >
> >>
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
> >> >        at
> >> >
> >>
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
> >> >        at
> >> >
> org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
> >> >        at
> >> >
> org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
> >> > 12/02/09 19:09:24 INFO
> >> client.HConnectionManager$HConnectionImplementation:
> >> > getMaster attempt 5 of 10 failed; retrying after sleep of 4000
> >> > java.net.ConnectException: Connection refused
> >> >        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
> >> >        at
> >> > sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
> >> >        at
> >> >
> >>
> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
> >> >        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
> >> >        at
> >> >
> >>
> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
> >> >        at
> >> >
> >>
> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
> >> >        at
> >> > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
> >> >        at
> >> > org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
> >> >        at $Proxy4.getProtocolVersion(Unknown Source)
> >> >        at
> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
> >> >        at
> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
> >> >        at
> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
> >> >        at
> >> >
> >>
> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
> >> >        at
> >> > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
> >> >        at
> >> >
> >>
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
> >> >        at
> >> >
> >>
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
> >> >        at
> >> >
> org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
> >> >        at
> >> >
> org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
> >> > 12/02/09 19:09:28 INFO
> >> client.HConnectionManager$HConnectionImplementation:
> >> > getMaster attempt 6 of 10 failed; retrying after sleep of 4000
> >> > java.net.ConnectException: Connection refused
> >> >        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
> >> >        at
> >> > sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
> >> >        at
> >> >
> >>
> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
> >> >        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
> >> >        at
> >> >
> >>
> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
> >> >        at
> >> >
> >>
> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
> >> >        at
> >> > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
> >> >        at
> >> > org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
> >> >        at $Proxy4.getProtocolVersion(Unknown Source)
> >> >        at
> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
> >> >        at
> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
> >> >        at
> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
> >> >        at
> >> >
> >>
> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
> >> >        at
> >> > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
> >> >        at
> >> >
> >>
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
> >> >        at
> >> >
> >>
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
> >> >        at
> >> >
> org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
> >> >        at
> >> >
> org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
> >> > 12/02/09 19:09:32 INFO
> >> client.HConnectionManager$HConnectionImplementation:
> >> > getMaster attempt 7 of 10 failed; retrying after sleep of 8000
> >> > java.net.ConnectException: Connection refused
> >> >        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
> >> >        at
> >> > sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
> >> >        at
> >> >
> >>
> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
> >> >        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
> >> >        at
> >> >
> >>
> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
> >> >        at
> >> >
> >>
> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
> >> >        at
> >> > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
> >> >        at
> >> > org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
> >> >        at $Proxy4.getProtocolVersion(Unknown Source)
> >> >        at
> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
> >> >        at
> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
> >> >        at
> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
> >> >        at
> >> >
> >>
> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
> >> >        at
> >> > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
> >> >        at
> >> >
> >>
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
> >> >        at
> >> >
> >>
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
> >> >        at
> >> >
> org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
> >> >        at
> >> >
> org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
> >> > 12/02/09 19:09:40 INFO
> >> client.HConnectionManager$HConnectionImplementation:
> >> > getMaster attempt 8 of 10 failed; retrying after sleep of 16000
> >> > java.net.ConnectException: Connection refused
> >> >        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
> >> >        at
> >> > sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
> >> >        at
> >> >
> >>
> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
> >> >        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
> >> >        at
> >> >
> >>
> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
> >> >        at
> >> >
> >>
> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
> >> >        at
> >> > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
> >> >        at
> >> > org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
> >> >        at $Proxy4.getProtocolVersion(Unknown Source)
> >> >        at
> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
> >> >        at
> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
> >> >        at
> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
> >> >        at
> >> >
> >>
> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
> >> >        at
> >> > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
> >> >        at
> >> >
> >>
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
> >> >        at
> >> >
> >>
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
> >> >        at
> >> >
> org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
> >> >        at
> >> >
> org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
> >> > 12/02/09 19:09:56 INFO
> >> client.HConnectionManager$HConnectionImplementation:
> >> > getMaster attempt 9 of 10 failed; no more retrying.
> >> > java.net.ConnectException: Connection refused
> >> >        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
> >> >        at
> >> > sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
> >> >        at
> >> >
> >>
> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
> >> >        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
> >> >        at
> >> >
> >>
> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
> >> >        at
> >> >
> >>
> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
> >> >        at
> >> > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
> >> >        at
> >> > org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
> >> >        at $Proxy4.getProtocolVersion(Unknown Source)
> >> >        at
> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
> >> >        at
> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
> >> >        at
> >> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
> >> >        at
> >> >
> >>
> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
> >> >        at
> >> > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
> >> >        at
> >> >
> >>
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
> >> >        at
> >> >
> >>
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
> >> >        at
> >> >
> org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
> >> >        at
> >> >
> org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
> >> > Exception in thread "main"
> >> > org.apache.hadoop.hbase.MasterNotRunningException: master:60000
> >> >        at
> >> >
> >>
> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:396)
> >> >        at
> >> > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
> >> >        at
> >> >
> >>
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
> >> >        at
> >> >
> >>
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
> >> >        at
> >> >
> org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
> >> >        at
> >> >
> org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
> >>
>

Re: Thrift on regionservers - won't connect

Posted by Jean-Daniel Cryans <jd...@apache.org>.
60010 is the web ui port by default, 60000 is the port the master uses
for communication. Clients (like thrift) and region servers don't rely
on a configured port because the master already posts it in zookeeper
along with its address (that's how your thrift server found it).

If your region servers are using master:60000 and they don't have any
problem connecting to a port that doesn't respond, we have a bigger
issue here :)

J-D

On Thu, Feb 9, 2012 at 11:38 AM, Tomas Tillery <to...@gmail.com> wrote:
> No, it isn't - as far as I know, it should be aiming for 60010, rather than
> 60000 (though HBase is also bound to 60000, there's no telnet response
> there). There's probably something obvious I'm not managing to see, but I
> can't find a way to redirect it to the proper port.
> Starting thrift with this install is a matter of invoking
> /etc/init.d/hadoop-hbase-thrift. Since it isn't invoked through the shell
> script, I'm not sure where to specify the port to connect to.
>
> On Thu, Feb 9, 2012 at 2:27 PM, Jean-Daniel Cryans <jd...@apache.org>wrote:
>
>> Connection refused is a java exception that usually means that either
>> there's nothing listening on the port or it's firewall'd. Is
>> "master:60000" really what you're expecting to see here? Should be
>> easy to "telnet master 60000" from that machien to really verify that
>> it works as expected.
>>
>> J-D
>>
>> On Thu, Feb 9, 2012 at 11:19 AM, Tomas Tillery <to...@gmail.com>
>> wrote:
>> > Hello, all
>> >
>> > It's my understanding that it's best practice to colocate thrift on each
>> of
>> > the regionservers, and use load balancing to spread queries between them.
>> > However, I can't seem to get thrift to bind from the regionservers (only
>> > from the hmaster).
>> >
>> > The machines have connectivity between them; regionservers connect and
>> > communicate with the hmaster without any problems.
>> >
>> > I'm running Hadoop 0.20.2-cdh3u2, HBase 0.90.4-cdh3u2.
>> >
>> > Thrift can run succesfully from the HMaster, but not from any of the
>> > datanode/regionservers
>> >
>> > What appears to be the relevant content from
>> > hbase-hbase-thrift-region-1.out below:
>> >
>> > 12/02/09 19:09:17 INFO
>> client.HConnectionManager$HConnectionImplementation:
>> > getMaster attempt 0 of 10 failed; retrying after sleep of 1000
>> > java.net.ConnectException: Connection refused
>> >        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>> >        at
>> > sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
>> >        at
>> >
>> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
>> >        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
>> >        at
>> >
>> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
>> >        at
>> >
>> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
>> >        at
>> > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
>> >        at
>> > org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
>> >        at $Proxy4.getProtocolVersion(Unknown Source)
>> >        at
>> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
>> >        at
>> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
>> >        at
>> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
>> >        at
>> >
>> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
>> >        at
>> > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
>> >        at
>> >
>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
>> >        at
>> >
>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
>> >        at
>> > org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
>> >        at
>> > org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
>> > 12/02/09 19:09:18 INFO
>> client.HConnectionManager$HConnectionImplementation:
>> > getMaster attempt 1 of 10 failed; retrying after sleep of 1000
>> > java.net.ConnectException: Connection refused
>> >        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>> >        at
>> > sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
>> >        at
>> >
>> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
>> >        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
>> >        at
>> >
>> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
>> >        at
>> >
>> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
>> >        at
>> > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
>> >        at
>> > org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
>> >        at $Proxy4.getProtocolVersion(Unknown Source)
>> >        at
>> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
>> >        at
>> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
>> >        at
>> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
>> >        at
>> >
>> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
>> >        at
>> > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
>> >        at
>> >
>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
>> >        at
>> >
>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
>> >        at
>> > org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
>> >        at
>> > org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
>> > 12/02/09 19:09:19 INFO
>> client.HConnectionManager$HConnectionImplementation:
>> > getMaster attempt 2 of 10 failed; retrying after sleep of 1000
>> > java.net.ConnectException: Connection refused
>> >        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>> >        at
>> > sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
>> >        at
>> >
>> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
>> >        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
>> >        at
>> >
>> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
>> >        at
>> >
>> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
>> >        at
>> > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
>> >        at
>> > org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
>> >        at $Proxy4.getProtocolVersion(Unknown Source)
>> >        at
>> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
>> >        at
>> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
>> >        at
>> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
>> >        at
>> >
>> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
>> >        at
>> > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
>> >        at
>> >
>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
>> >        at
>> >
>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
>> >        at
>> > org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
>> >        at
>> > org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
>> > 12/02/09 19:09:20 INFO
>> client.HConnectionManager$HConnectionImplementation:
>> > getMaster attempt 3 of 10 failed; retrying after sleep of 2000
>> > java.net.ConnectException: Connection refused
>> >        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>> >        at
>> > sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
>> >        at
>> >
>> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
>> >        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
>> >        at
>> >
>> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
>> >        at
>> >
>> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
>> >        at
>> > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
>> >        at
>> > org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
>> >        at $Proxy4.getProtocolVersion(Unknown Source)
>> >        at
>> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
>> >        at
>> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
>> >        at
>> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
>> >        at
>> >
>> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
>> >        at
>> > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
>> >        at
>> >
>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
>> >        at
>> >
>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
>> >        at
>> > org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
>> >        at
>> > org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
>> > 12/02/09 19:09:22 INFO
>> client.HConnectionManager$HConnectionImplementation:
>> > getMaster attempt 4 of 10 failed; retrying after sleep of 2000
>> > java.net.ConnectException: Connection refused
>> >        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>> >        at
>> > sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
>> >        at
>> >
>> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
>> >        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
>> >        at
>> >
>> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
>> >        at
>> >
>> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
>> >        at
>> > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
>> >        at
>> > org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
>> >        at $Proxy4.getProtocolVersion(Unknown Source)
>> >        at
>> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
>> >        at
>> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
>> >        at
>> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
>> >        at
>> >
>> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
>> >        at
>> > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
>> >        at
>> >
>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
>> >        at
>> >
>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
>> >        at
>> > org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
>> >        at
>> > org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
>> > 12/02/09 19:09:24 INFO
>> client.HConnectionManager$HConnectionImplementation:
>> > getMaster attempt 5 of 10 failed; retrying after sleep of 4000
>> > java.net.ConnectException: Connection refused
>> >        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>> >        at
>> > sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
>> >        at
>> >
>> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
>> >        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
>> >        at
>> >
>> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
>> >        at
>> >
>> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
>> >        at
>> > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
>> >        at
>> > org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
>> >        at $Proxy4.getProtocolVersion(Unknown Source)
>> >        at
>> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
>> >        at
>> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
>> >        at
>> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
>> >        at
>> >
>> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
>> >        at
>> > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
>> >        at
>> >
>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
>> >        at
>> >
>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
>> >        at
>> > org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
>> >        at
>> > org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
>> > 12/02/09 19:09:28 INFO
>> client.HConnectionManager$HConnectionImplementation:
>> > getMaster attempt 6 of 10 failed; retrying after sleep of 4000
>> > java.net.ConnectException: Connection refused
>> >        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>> >        at
>> > sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
>> >        at
>> >
>> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
>> >        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
>> >        at
>> >
>> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
>> >        at
>> >
>> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
>> >        at
>> > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
>> >        at
>> > org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
>> >        at $Proxy4.getProtocolVersion(Unknown Source)
>> >        at
>> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
>> >        at
>> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
>> >        at
>> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
>> >        at
>> >
>> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
>> >        at
>> > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
>> >        at
>> >
>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
>> >        at
>> >
>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
>> >        at
>> > org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
>> >        at
>> > org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
>> > 12/02/09 19:09:32 INFO
>> client.HConnectionManager$HConnectionImplementation:
>> > getMaster attempt 7 of 10 failed; retrying after sleep of 8000
>> > java.net.ConnectException: Connection refused
>> >        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>> >        at
>> > sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
>> >        at
>> >
>> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
>> >        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
>> >        at
>> >
>> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
>> >        at
>> >
>> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
>> >        at
>> > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
>> >        at
>> > org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
>> >        at $Proxy4.getProtocolVersion(Unknown Source)
>> >        at
>> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
>> >        at
>> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
>> >        at
>> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
>> >        at
>> >
>> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
>> >        at
>> > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
>> >        at
>> >
>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
>> >        at
>> >
>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
>> >        at
>> > org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
>> >        at
>> > org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
>> > 12/02/09 19:09:40 INFO
>> client.HConnectionManager$HConnectionImplementation:
>> > getMaster attempt 8 of 10 failed; retrying after sleep of 16000
>> > java.net.ConnectException: Connection refused
>> >        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>> >        at
>> > sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
>> >        at
>> >
>> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
>> >        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
>> >        at
>> >
>> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
>> >        at
>> >
>> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
>> >        at
>> > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
>> >        at
>> > org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
>> >        at $Proxy4.getProtocolVersion(Unknown Source)
>> >        at
>> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
>> >        at
>> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
>> >        at
>> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
>> >        at
>> >
>> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
>> >        at
>> > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
>> >        at
>> >
>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
>> >        at
>> >
>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
>> >        at
>> > org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
>> >        at
>> > org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
>> > 12/02/09 19:09:56 INFO
>> client.HConnectionManager$HConnectionImplementation:
>> > getMaster attempt 9 of 10 failed; no more retrying.
>> > java.net.ConnectException: Connection refused
>> >        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>> >        at
>> > sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
>> >        at
>> >
>> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
>> >        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
>> >        at
>> >
>> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
>> >        at
>> >
>> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
>> >        at
>> > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
>> >        at
>> > org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
>> >        at $Proxy4.getProtocolVersion(Unknown Source)
>> >        at
>> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
>> >        at
>> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
>> >        at
>> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
>> >        at
>> >
>> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
>> >        at
>> > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
>> >        at
>> >
>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
>> >        at
>> >
>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
>> >        at
>> > org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
>> >        at
>> > org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
>> > Exception in thread "main"
>> > org.apache.hadoop.hbase.MasterNotRunningException: master:60000
>> >        at
>> >
>> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:396)
>> >        at
>> > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
>> >        at
>> >
>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
>> >        at
>> >
>> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
>> >        at
>> > org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
>> >        at
>> > org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
>>

Re: Thrift on regionservers - won't connect

Posted by Tomas Tillery <to...@gmail.com>.
No, it isn't - as far as I know, it should be aiming for 60010, rather than
60000 (though HBase is also bound to 60000, there's no telnet response
there). There's probably something obvious I'm not managing to see, but I
can't find a way to redirect it to the proper port.
Starting thrift with this install is a matter of invoking
/etc/init.d/hadoop-hbase-thrift. Since it isn't invoked through the shell
script, I'm not sure where to specify the port to connect to.

On Thu, Feb 9, 2012 at 2:27 PM, Jean-Daniel Cryans <jd...@apache.org>wrote:

> Connection refused is a java exception that usually means that either
> there's nothing listening on the port or it's firewall'd. Is
> "master:60000" really what you're expecting to see here? Should be
> easy to "telnet master 60000" from that machien to really verify that
> it works as expected.
>
> J-D
>
> On Thu, Feb 9, 2012 at 11:19 AM, Tomas Tillery <to...@gmail.com>
> wrote:
> > Hello, all
> >
> > It's my understanding that it's best practice to colocate thrift on each
> of
> > the regionservers, and use load balancing to spread queries between them.
> > However, I can't seem to get thrift to bind from the regionservers (only
> > from the hmaster).
> >
> > The machines have connectivity between them; regionservers connect and
> > communicate with the hmaster without any problems.
> >
> > I'm running Hadoop 0.20.2-cdh3u2, HBase 0.90.4-cdh3u2.
> >
> > Thrift can run succesfully from the HMaster, but not from any of the
> > datanode/regionservers
> >
> > What appears to be the relevant content from
> > hbase-hbase-thrift-region-1.out below:
> >
> > 12/02/09 19:09:17 INFO
> client.HConnectionManager$HConnectionImplementation:
> > getMaster attempt 0 of 10 failed; retrying after sleep of 1000
> > java.net.ConnectException: Connection refused
> >        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
> >        at
> > sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
> >        at
> >
> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
> >        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
> >        at
> >
> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
> >        at
> >
> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
> >        at
> > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
> >        at
> > org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
> >        at $Proxy4.getProtocolVersion(Unknown Source)
> >        at
> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
> >        at
> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
> >        at
> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
> >        at
> >
> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
> >        at
> > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
> >        at
> >
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
> >        at
> >
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
> >        at
> > org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
> >        at
> > org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
> > 12/02/09 19:09:18 INFO
> client.HConnectionManager$HConnectionImplementation:
> > getMaster attempt 1 of 10 failed; retrying after sleep of 1000
> > java.net.ConnectException: Connection refused
> >        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
> >        at
> > sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
> >        at
> >
> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
> >        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
> >        at
> >
> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
> >        at
> >
> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
> >        at
> > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
> >        at
> > org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
> >        at $Proxy4.getProtocolVersion(Unknown Source)
> >        at
> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
> >        at
> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
> >        at
> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
> >        at
> >
> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
> >        at
> > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
> >        at
> >
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
> >        at
> >
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
> >        at
> > org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
> >        at
> > org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
> > 12/02/09 19:09:19 INFO
> client.HConnectionManager$HConnectionImplementation:
> > getMaster attempt 2 of 10 failed; retrying after sleep of 1000
> > java.net.ConnectException: Connection refused
> >        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
> >        at
> > sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
> >        at
> >
> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
> >        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
> >        at
> >
> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
> >        at
> >
> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
> >        at
> > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
> >        at
> > org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
> >        at $Proxy4.getProtocolVersion(Unknown Source)
> >        at
> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
> >        at
> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
> >        at
> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
> >        at
> >
> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
> >        at
> > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
> >        at
> >
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
> >        at
> >
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
> >        at
> > org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
> >        at
> > org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
> > 12/02/09 19:09:20 INFO
> client.HConnectionManager$HConnectionImplementation:
> > getMaster attempt 3 of 10 failed; retrying after sleep of 2000
> > java.net.ConnectException: Connection refused
> >        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
> >        at
> > sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
> >        at
> >
> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
> >        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
> >        at
> >
> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
> >        at
> >
> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
> >        at
> > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
> >        at
> > org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
> >        at $Proxy4.getProtocolVersion(Unknown Source)
> >        at
> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
> >        at
> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
> >        at
> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
> >        at
> >
> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
> >        at
> > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
> >        at
> >
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
> >        at
> >
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
> >        at
> > org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
> >        at
> > org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
> > 12/02/09 19:09:22 INFO
> client.HConnectionManager$HConnectionImplementation:
> > getMaster attempt 4 of 10 failed; retrying after sleep of 2000
> > java.net.ConnectException: Connection refused
> >        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
> >        at
> > sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
> >        at
> >
> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
> >        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
> >        at
> >
> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
> >        at
> >
> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
> >        at
> > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
> >        at
> > org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
> >        at $Proxy4.getProtocolVersion(Unknown Source)
> >        at
> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
> >        at
> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
> >        at
> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
> >        at
> >
> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
> >        at
> > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
> >        at
> >
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
> >        at
> >
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
> >        at
> > org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
> >        at
> > org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
> > 12/02/09 19:09:24 INFO
> client.HConnectionManager$HConnectionImplementation:
> > getMaster attempt 5 of 10 failed; retrying after sleep of 4000
> > java.net.ConnectException: Connection refused
> >        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
> >        at
> > sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
> >        at
> >
> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
> >        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
> >        at
> >
> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
> >        at
> >
> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
> >        at
> > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
> >        at
> > org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
> >        at $Proxy4.getProtocolVersion(Unknown Source)
> >        at
> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
> >        at
> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
> >        at
> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
> >        at
> >
> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
> >        at
> > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
> >        at
> >
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
> >        at
> >
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
> >        at
> > org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
> >        at
> > org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
> > 12/02/09 19:09:28 INFO
> client.HConnectionManager$HConnectionImplementation:
> > getMaster attempt 6 of 10 failed; retrying after sleep of 4000
> > java.net.ConnectException: Connection refused
> >        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
> >        at
> > sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
> >        at
> >
> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
> >        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
> >        at
> >
> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
> >        at
> >
> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
> >        at
> > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
> >        at
> > org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
> >        at $Proxy4.getProtocolVersion(Unknown Source)
> >        at
> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
> >        at
> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
> >        at
> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
> >        at
> >
> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
> >        at
> > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
> >        at
> >
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
> >        at
> >
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
> >        at
> > org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
> >        at
> > org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
> > 12/02/09 19:09:32 INFO
> client.HConnectionManager$HConnectionImplementation:
> > getMaster attempt 7 of 10 failed; retrying after sleep of 8000
> > java.net.ConnectException: Connection refused
> >        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
> >        at
> > sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
> >        at
> >
> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
> >        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
> >        at
> >
> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
> >        at
> >
> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
> >        at
> > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
> >        at
> > org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
> >        at $Proxy4.getProtocolVersion(Unknown Source)
> >        at
> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
> >        at
> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
> >        at
> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
> >        at
> >
> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
> >        at
> > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
> >        at
> >
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
> >        at
> >
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
> >        at
> > org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
> >        at
> > org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
> > 12/02/09 19:09:40 INFO
> client.HConnectionManager$HConnectionImplementation:
> > getMaster attempt 8 of 10 failed; retrying after sleep of 16000
> > java.net.ConnectException: Connection refused
> >        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
> >        at
> > sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
> >        at
> >
> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
> >        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
> >        at
> >
> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
> >        at
> >
> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
> >        at
> > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
> >        at
> > org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
> >        at $Proxy4.getProtocolVersion(Unknown Source)
> >        at
> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
> >        at
> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
> >        at
> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
> >        at
> >
> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
> >        at
> > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
> >        at
> >
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
> >        at
> >
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
> >        at
> > org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
> >        at
> > org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
> > 12/02/09 19:09:56 INFO
> client.HConnectionManager$HConnectionImplementation:
> > getMaster attempt 9 of 10 failed; no more retrying.
> > java.net.ConnectException: Connection refused
> >        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
> >        at
> > sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
> >        at
> >
> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
> >        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
> >        at
> >
> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
> >        at
> >
> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
> >        at
> > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
> >        at
> > org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
> >        at $Proxy4.getProtocolVersion(Unknown Source)
> >        at
> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
> >        at
> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
> >        at
> org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
> >        at
> >
> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
> >        at
> > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
> >        at
> >
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
> >        at
> >
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
> >        at
> > org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
> >        at
> > org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
> > Exception in thread "main"
> > org.apache.hadoop.hbase.MasterNotRunningException: master:60000
> >        at
> >
> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:396)
> >        at
> > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
> >        at
> >
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
> >        at
> >
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
> >        at
> > org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
> >        at
> > org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
>

Re: Thrift on regionservers - won't connect

Posted by Jean-Daniel Cryans <jd...@apache.org>.
Connection refused is a java exception that usually means that either
there's nothing listening on the port or it's firewall'd. Is
"master:60000" really what you're expecting to see here? Should be
easy to "telnet master 60000" from that machien to really verify that
it works as expected.

J-D

On Thu, Feb 9, 2012 at 11:19 AM, Tomas Tillery <to...@gmail.com> wrote:
> Hello, all
>
> It's my understanding that it's best practice to colocate thrift on each of
> the regionservers, and use load balancing to spread queries between them.
> However, I can't seem to get thrift to bind from the regionservers (only
> from the hmaster).
>
> The machines have connectivity between them; regionservers connect and
> communicate with the hmaster without any problems.
>
> I'm running Hadoop 0.20.2-cdh3u2, HBase 0.90.4-cdh3u2.
>
> Thrift can run succesfully from the HMaster, but not from any of the
> datanode/regionservers
>
> What appears to be the relevant content from
> hbase-hbase-thrift-region-1.out below:
>
> 12/02/09 19:09:17 INFO client.HConnectionManager$HConnectionImplementation:
> getMaster attempt 0 of 10 failed; retrying after sleep of 1000
> java.net.ConnectException: Connection refused
>        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>        at
> sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
>        at
> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
>        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
>        at
> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
>        at
> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
>        at
> org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
>        at
> org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
>        at $Proxy4.getProtocolVersion(Unknown Source)
>        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
>        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
>        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
>        at
> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
>        at
> org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
>        at
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
>        at
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
>        at
> org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
>        at
> org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
> 12/02/09 19:09:18 INFO client.HConnectionManager$HConnectionImplementation:
> getMaster attempt 1 of 10 failed; retrying after sleep of 1000
> java.net.ConnectException: Connection refused
>        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>        at
> sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
>        at
> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
>        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
>        at
> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
>        at
> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
>        at
> org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
>        at
> org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
>        at $Proxy4.getProtocolVersion(Unknown Source)
>        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
>        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
>        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
>        at
> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
>        at
> org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
>        at
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
>        at
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
>        at
> org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
>        at
> org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
> 12/02/09 19:09:19 INFO client.HConnectionManager$HConnectionImplementation:
> getMaster attempt 2 of 10 failed; retrying after sleep of 1000
> java.net.ConnectException: Connection refused
>        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>        at
> sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
>        at
> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
>        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
>        at
> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
>        at
> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
>        at
> org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
>        at
> org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
>        at $Proxy4.getProtocolVersion(Unknown Source)
>        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
>        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
>        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
>        at
> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
>        at
> org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
>        at
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
>        at
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
>        at
> org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
>        at
> org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
> 12/02/09 19:09:20 INFO client.HConnectionManager$HConnectionImplementation:
> getMaster attempt 3 of 10 failed; retrying after sleep of 2000
> java.net.ConnectException: Connection refused
>        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>        at
> sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
>        at
> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
>        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
>        at
> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
>        at
> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
>        at
> org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
>        at
> org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
>        at $Proxy4.getProtocolVersion(Unknown Source)
>        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
>        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
>        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
>        at
> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
>        at
> org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
>        at
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
>        at
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
>        at
> org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
>        at
> org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
> 12/02/09 19:09:22 INFO client.HConnectionManager$HConnectionImplementation:
> getMaster attempt 4 of 10 failed; retrying after sleep of 2000
> java.net.ConnectException: Connection refused
>        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>        at
> sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
>        at
> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
>        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
>        at
> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
>        at
> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
>        at
> org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
>        at
> org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
>        at $Proxy4.getProtocolVersion(Unknown Source)
>        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
>        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
>        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
>        at
> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
>        at
> org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
>        at
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
>        at
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
>        at
> org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
>        at
> org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
> 12/02/09 19:09:24 INFO client.HConnectionManager$HConnectionImplementation:
> getMaster attempt 5 of 10 failed; retrying after sleep of 4000
> java.net.ConnectException: Connection refused
>        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>        at
> sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
>        at
> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
>        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
>        at
> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
>        at
> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
>        at
> org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
>        at
> org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
>        at $Proxy4.getProtocolVersion(Unknown Source)
>        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
>        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
>        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
>        at
> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
>        at
> org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
>        at
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
>        at
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
>        at
> org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
>        at
> org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
> 12/02/09 19:09:28 INFO client.HConnectionManager$HConnectionImplementation:
> getMaster attempt 6 of 10 failed; retrying after sleep of 4000
> java.net.ConnectException: Connection refused
>        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>        at
> sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
>        at
> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
>        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
>        at
> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
>        at
> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
>        at
> org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
>        at
> org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
>        at $Proxy4.getProtocolVersion(Unknown Source)
>        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
>        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
>        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
>        at
> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
>        at
> org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
>        at
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
>        at
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
>        at
> org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
>        at
> org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
> 12/02/09 19:09:32 INFO client.HConnectionManager$HConnectionImplementation:
> getMaster attempt 7 of 10 failed; retrying after sleep of 8000
> java.net.ConnectException: Connection refused
>        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>        at
> sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
>        at
> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
>        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
>        at
> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
>        at
> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
>        at
> org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
>        at
> org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
>        at $Proxy4.getProtocolVersion(Unknown Source)
>        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
>        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
>        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
>        at
> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
>        at
> org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
>        at
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
>        at
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
>        at
> org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
>        at
> org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
> 12/02/09 19:09:40 INFO client.HConnectionManager$HConnectionImplementation:
> getMaster attempt 8 of 10 failed; retrying after sleep of 16000
> java.net.ConnectException: Connection refused
>        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>        at
> sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
>        at
> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
>        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
>        at
> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
>        at
> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
>        at
> org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
>        at
> org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
>        at $Proxy4.getProtocolVersion(Unknown Source)
>        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
>        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
>        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
>        at
> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
>        at
> org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
>        at
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
>        at
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
>        at
> org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
>        at
> org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
> 12/02/09 19:09:56 INFO client.HConnectionManager$HConnectionImplementation:
> getMaster attempt 9 of 10 failed; no more retrying.
> java.net.ConnectException: Connection refused
>        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>        at
> sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574)
>        at
> org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
>        at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408)
>        at
> org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328)
>        at
> org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883)
>        at
> org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750)
>        at
> org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257)
>        at $Proxy4.getProtocolVersion(Unknown Source)
>        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419)
>        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393)
>        at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444)
>        at
> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360)
>        at
> org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
>        at
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
>        at
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
>        at
> org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
>        at
> org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
> Exception in thread "main"
> org.apache.hadoop.hbase.MasterNotRunningException: master:60000
>        at
> org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:396)
>        at
> org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94)
>        at
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194)
>        at
> org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188)
>        at
> org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870)
>        at
> org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)