You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@hbase.apache.org by Gao Zavier <za...@gmail.com> on 2012/11/06 10:36:41 UTC
HConnectionManager may leaks HConnection ( and zookeeper connections)
In HConnectionManager, there are several functions to delete connections.
All of them work fine except deleteAllConnections
public static void deleteAllConnections(boolean stopProxy) {
synchronized (HBASE_INSTANCES) {
Set<HConnectionKey> connectionKeys = new HashSet<HConnectionKey>();
connectionKeys.addAll(HBASE_INSTANCES.keySet());
for (HConnectionKey connectionKey : connectionKeys) {
*deleteConnection(connectionKey, stopProxy, false);*
}
*HBASE_INSTANCES.clear();*
}
}
It calls deleteConnection for each connectionKey and clear map *
HBASE_INSTANCES*.
The issue is the HConnectionImplemention and the nested zookeeper
connection may not clear (for the refcount is not zero).
my client code is:
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.HConnectionManager;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.client.HConnection;
public class ConnectionTest {
public static Result get(Configuration conf,
String tableName, String rowKey) throws IOException
{
HTable table = new HTable(conf, tableName);
Get get = new Get(Bytes.toBytes(rowKey));
get.setMaxVersions();
return getResult(table, get);
}
public static Result getResult(HTable table,
Get get) throws IOException
{
Result result = table.get(get);
HConnectionManager.deleteAllConnections(true);
return result;
}
public static void main(String[] args) throws IOException, Exception
{
for (int i = 0; i < 100; ++i) {
System.out.println("Connect to Hbase, attempts " + i + "
times.");
Configuration icfg = new Configuration();
icfg.setStrings("hbase.zookeeper.quorum",
"192.168.23.7,192.168.23.8,192.168.23.9");
icfg.setStrings("zookeeper.znode.parent", "/hbase");
Configuration conf = HBaseConfiguration.create(icfg);
String tableName = "QuickExpand";
String rowKey = "AbnormalDeal_600202";
Result result = ConnectionTest.get(conf, tableName, rowKey);
Thread.currentThread().sleep(100);
}
}
}
and on my testing linuxbox, command: netstat -anp | grep ":2181" | wc -l
shows the connections continue to increase.
Re: HConnectionManager may leaks HConnection ( and zookeeper connections)
Posted by Gao Zavier <za...@gmail.com>.
I am using 0.92.0, r15057
Regards
Zavier
On Tue, Nov 6, 2012 at 5:39 PM, ramkrishna vasudevan <
ramkrishna.s.vasudevan@gmail.com> wrote:
> Which version of hbase are you using ?
>
> Regards
> Ram
>
> On Tue, Nov 6, 2012 at 3:06 PM, Gao Zavier <za...@gmail.com> wrote:
>
> > In HConnectionManager, there are several functions to delete connections.
> > All of them work fine except deleteAllConnections
> >
> > public static void deleteAllConnections(boolean stopProxy) {
> > synchronized (HBASE_INSTANCES) {
> > Set<HConnectionKey> connectionKeys = new HashSet<HConnectionKey>();
> > connectionKeys.addAll(HBASE_INSTANCES.keySet());
> > for (HConnectionKey connectionKey : connectionKeys) {
> > *deleteConnection(connectionKey, stopProxy, false);*
> > }
> > *HBASE_INSTANCES.clear();*
> > }
> > }
> >
> > It calls deleteConnection for each connectionKey and clear map *
> > HBASE_INSTANCES*.
> > The issue is the HConnectionImplemention and the nested zookeeper
> > connection may not clear (for the refcount is not zero).
> >
> > my client code is:
> >
> > import java.io.IOException;
> >
> > import org.apache.hadoop.conf.Configuration;
> > import org.apache.hadoop.hbase.HBaseConfiguration;
> >
> > import org.apache.hadoop.hbase.client.HTable;
> > import org.apache.hadoop.hbase.client.Get;
> > import org.apache.hadoop.hbase.client.Result;
> > import org.apache.hadoop.hbase.client.HConnectionManager;
> > import org.apache.hadoop.hbase.util.Bytes;
> >
> > import org.apache.hadoop.hbase.client.HConnection;
> >
> > public class ConnectionTest {
> > public static Result get(Configuration conf,
> > String tableName, String rowKey) throws IOException
> > {
> > HTable table = new HTable(conf, tableName);
> > Get get = new Get(Bytes.toBytes(rowKey));
> > get.setMaxVersions();
> > return getResult(table, get);
> > }
> >
> > public static Result getResult(HTable table,
> > Get get) throws IOException
> > {
> > Result result = table.get(get);
> >
> > HConnectionManager.deleteAllConnections(true);
> >
> > return result;
> > }
> >
> > public static void main(String[] args) throws IOException, Exception
> > {
> > for (int i = 0; i < 100; ++i) {
> > System.out.println("Connect to Hbase, attempts " + i + "
> > times.");
> > Configuration icfg = new Configuration();
> > icfg.setStrings("hbase.zookeeper.quorum",
> > "192.168.23.7,192.168.23.8,192.168.23.9");
> > icfg.setStrings("zookeeper.znode.parent", "/hbase");
> > Configuration conf = HBaseConfiguration.create(icfg);
> >
> > String tableName = "QuickExpand";
> > String rowKey = "AbnormalDeal_600202";
> >
> > Result result = ConnectionTest.get(conf, tableName, rowKey);
> > Thread.currentThread().sleep(100);
> > }
> > }
> > }
> >
> >
> > and on my testing linuxbox, command: netstat -anp | grep ":2181" | wc -l
> > shows the connections continue to increase.
> >
>
Re: HConnectionManager may leaks HConnection ( and zookeeper connections)
Posted by ramkrishna vasudevan <ra...@gmail.com>.
Which version of hbase are you using ?
Regards
Ram
On Tue, Nov 6, 2012 at 3:06 PM, Gao Zavier <za...@gmail.com> wrote:
> In HConnectionManager, there are several functions to delete connections.
> All of them work fine except deleteAllConnections
>
> public static void deleteAllConnections(boolean stopProxy) {
> synchronized (HBASE_INSTANCES) {
> Set<HConnectionKey> connectionKeys = new HashSet<HConnectionKey>();
> connectionKeys.addAll(HBASE_INSTANCES.keySet());
> for (HConnectionKey connectionKey : connectionKeys) {
> *deleteConnection(connectionKey, stopProxy, false);*
> }
> *HBASE_INSTANCES.clear();*
> }
> }
>
> It calls deleteConnection for each connectionKey and clear map *
> HBASE_INSTANCES*.
> The issue is the HConnectionImplemention and the nested zookeeper
> connection may not clear (for the refcount is not zero).
>
> my client code is:
>
> import java.io.IOException;
>
> import org.apache.hadoop.conf.Configuration;
> import org.apache.hadoop.hbase.HBaseConfiguration;
>
> import org.apache.hadoop.hbase.client.HTable;
> import org.apache.hadoop.hbase.client.Get;
> import org.apache.hadoop.hbase.client.Result;
> import org.apache.hadoop.hbase.client.HConnectionManager;
> import org.apache.hadoop.hbase.util.Bytes;
>
> import org.apache.hadoop.hbase.client.HConnection;
>
> public class ConnectionTest {
> public static Result get(Configuration conf,
> String tableName, String rowKey) throws IOException
> {
> HTable table = new HTable(conf, tableName);
> Get get = new Get(Bytes.toBytes(rowKey));
> get.setMaxVersions();
> return getResult(table, get);
> }
>
> public static Result getResult(HTable table,
> Get get) throws IOException
> {
> Result result = table.get(get);
>
> HConnectionManager.deleteAllConnections(true);
>
> return result;
> }
>
> public static void main(String[] args) throws IOException, Exception
> {
> for (int i = 0; i < 100; ++i) {
> System.out.println("Connect to Hbase, attempts " + i + "
> times.");
> Configuration icfg = new Configuration();
> icfg.setStrings("hbase.zookeeper.quorum",
> "192.168.23.7,192.168.23.8,192.168.23.9");
> icfg.setStrings("zookeeper.znode.parent", "/hbase");
> Configuration conf = HBaseConfiguration.create(icfg);
>
> String tableName = "QuickExpand";
> String rowKey = "AbnormalDeal_600202";
>
> Result result = ConnectionTest.get(conf, tableName, rowKey);
> Thread.currentThread().sleep(100);
> }
> }
> }
>
>
> and on my testing linuxbox, command: netstat -anp | grep ":2181" | wc -l
> shows the connections continue to increase.
>