You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@hbase.apache.org by elsif <el...@gmail.com> on 2009/09/10 03:07:59 UTC
Hbase 0.20.0 threading issue
Hello. We are trying to use Hbase in a threaded application and are
encountering an issue with the ZooKeeper connection being lost:
org.apache.hadoop.hbase.client.NoServerForRegionException: Timed out
trying to locate root region
The following sample program returns 10 results and 10 of the above
errors. Code was compiled against the 0.20.0 svn fork and the
pre-compiled Hadoop 0.20.0 release.
import java.io.IOException;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.util.Bytes;
// create 'fc_test', {NAME => 'json', VERSIONS => 1}, {NAME => 'data', VERSIONS =>1}
// put 'fc_test', 'testing123', 'json:', "[1,2,3]"
public class TableTest extends Thread {
public void run() {
try {
String key = "testing123";
HBaseConfiguration conf = new HBaseConfiguration();
HTable table = new HTable(conf, "fc_test");
Get get = new Get(Bytes.toBytes(key));
get.addFamily(Bytes.toBytes("json"));
Result result = table.get(get);
KeyValue[] kvs = result.raw();
System.out.println(Bytes.toString(kvs[0].getValue()));
} catch (Exception e) {
System.out.println(e.toString());
}
try {
Thread.currentThread().sleep(1000);
} catch (Exception e) {
}
}
public static void main(String args[]) throws IOException {
Thread t = null;
int i;
for (i = 0; i< 20; i++) {
t = new TableTest();
t.start();
}
// wait for last thread
try {
t.join();
} catch (InterruptedException e) {
}
}
}
Re: Hbase 0.20.0 threading issue
Posted by stack <st...@duboce.net>.
No worries.
I meant to say in my previous message thank you for putting together code to
illustrate the problem you were seeing. Helps.
St.Ack
On Thu, Sep 10, 2009 at 3:28 PM, elsif <el...@gmail.com> wrote:
> Sorry, I rested against the older version. Using a single static
> instance did indeed fix the issue.
>
> Thanks for your help,
> elsif
>
> stack wrote:
> > I tried your program with count set to 1000 threads (going against a
> > standalone hbase) and all threads got answers. Something else must be
> up.
> > Please tell me more about your setup.
> > Thanks,
> > St.Ack
> >
> >
> >
>
>
Re: Hbase 0.20.0 threading issue
Posted by elsif <el...@gmail.com>.
Sorry, I rested against the older version. Using a single static
instance did indeed fix the issue.
Thanks for your help,
elsif
stack wrote:
> I tried your program with count set to 1000 threads (going against a
> standalone hbase) and all threads got answers. Something else must be up.
> Please tell me more about your setup.
> Thanks,
> St.Ack
>
>
>
Re: Hbase 0.20.0 threading issue
Posted by stack <st...@duboce.net>.
I tried your program with count set to 1000 threads (going against a
standalone hbase) and all threads got answers. Something else must be up.
Please tell me more about your setup.
Thanks,
St.Ack
P.S. Here is my output (I added index as thread name and emitted that as
first thing on line):
09/09/10 13:36:34 INFO zookeeper.ZooKeeper: Client
environment:zookeeper.version=3.2.1-808558, built on 08/27/2009 18:48 GMT
09/09/10 13:36:34 INFO zookeeper.ZooKeeper: Client environment:host.name
=aron
09/09/10 13:36:34 INFO zookeeper.ZooKeeper: Client
environment:java.version=1.6.0_16
09/09/10 13:36:34 INFO zookeeper.ZooKeeper: Client
environment:java.vendor=Sun Microsystems Inc.
09/09/10 13:36:34 INFO zookeeper.ZooKeeper: Client
environment:java.home=/usr/lib/jvm/java-6-sun-1.6.0.16/jre
09/09/10 13:36:34 INFO zookeeper.ZooKeeper: Client
environment:java.class.path=/home/stack/checkouts/workspace/svn_hbase_trunk/bin/../conf:/usr/lib/jvm/java-6-sun//lib/tools.jar:/home/stack/checkouts/workspace/svn_hbase_trunk/bin/../build/classes:/home/stack/checkouts/workspace/svn_hbase_trunk/bin/../build/test:/home/stack/checkouts/workspace/svn_hbase_trunk/bin/../build:/home/stack/checkouts/workspace/svn_hbase_trunk/bin/../lib/AgileJSON-2009-03-30.jar:/home/stack/checkouts/workspace/svn_hbase_trunk/bin/../lib/commons-cli-2.0-SNAPSHOT.jar:/home/stack/checkouts/workspace/svn_hbase_trunk/bin/../lib/commons-el-from-jetty-5.1.4.jar:/home/stack/checkouts/workspace/svn_hbase_trunk/bin/../lib/commons-httpclient-3.0.1.jar:/home/stack/checkouts/workspace/svn_hbase_trunk/bin/../lib/commons-logging-1.0.4.jar:/home/stack/checkouts/workspace/svn_hbase_trunk/bin/../lib/commons-logging-api-1.0.4.jar:/home/stack/checkouts/workspace/svn_hbase_trunk/bin/../lib/commons-math-1.1.jar:/home/stack/checkouts/workspace/svn_hbase_trunk/bin/../lib/hadoop-0.20.1-hdfs127-core.jar:/home/stack/checkouts/workspace/svn_hbase_trunk/bin/../lib/hadoop-0.20.1-test.jar:/home/stack/checkouts/workspace/svn_hbase_trunk/bin/../lib/jasper-compiler-5.5.12.jar:/home/stack/checkouts/workspace/svn_hbase_trunk/bin/../lib/jasper-runtime-5.5.12.jar:/home/stack/checkouts/workspace/svn_hbase_trunk/bin/../lib/jetty-6.1.14.jar:/home/stack/checkouts/workspace/svn_hbase_trunk/bin/../lib/jetty-util-6.1.14.jar:/home/stack/checkouts/workspace/svn_hbase_trunk/bin/../lib/jruby-complete-1.3.1.jar:/home/stack/checkouts/workspace/svn_hbase_trunk/bin/../lib/json.jar:/home/stack/checkouts/workspace/svn_hbase_trunk/bin/../lib/junit-4.7.jar:/home/stack/checkouts/workspace/svn_hbase_trunk/bin/../lib/libthrift-r771587.jar:/home/stack/checkouts/workspace/svn_hbase_trunk/bin/../lib/log4j-1.2.15.jar:/home/stack/checkouts/workspace/svn_hbase_trunk/bin/../lib/lucene-core-2.2.0.jar:/home/stack/checkouts/workspace/svn_hbase_trunk/bin/../lib/servlet-api-2.5-6.1.14.jar:/home/stack/checkouts/workspace/svn_hbase_trunk/bin/../lib/xmlenc-0.52.jar:/home/stack/checkouts/workspace/svn_hbase_trunk/bin/../lib/zookeeper-3.2.1.jar:/home/stack/checkouts/workspace/svn_hbase_trunk/bin/../lib/jsp-2.1/jsp-2.1.jar:/home/stack/checkouts/workspace/svn_hbase_trunk/bin/../lib/jsp-2.1/jsp-api-2.1.jar
09/09/10 13:36:34 INFO zookeeper.ZooKeeper: Client
environment:java.library.path=/home/stack/checkouts/workspace/svn_hbase_trunk/bin/../lib/native/Linux-amd64-64
09/09/10 13:36:34 INFO zookeeper.ZooKeeper: Client
environment:java.io.tmpdir=/tmp
09/09/10 13:36:34 INFO zookeeper.ZooKeeper: Client
environment:java.compiler=<NA>
09/09/10 13:36:34 INFO zookeeper.ZooKeeper: Client environment:os.name=Linux
stack@aron:~/checkouts/workspace/svn_hbase_trunk$ head -40 /tmp/test.log
09/09/10 13:36:34 INFO zookeeper.ZooKeeper: Client
environment:zookeeper.version=3.2.1-808558, built on 08/27/2009 18:48 GMT
09/09/10 13:36:34 INFO zookeeper.ZooKeeper: Client environment:host.name
=aron
09/09/10 13:36:34 INFO zookeeper.ZooKeeper: Client
environment:java.version=1.6.0_16
09/09/10 13:36:34 INFO zookeeper.ZooKeeper: Client
environment:java.vendor=Sun Microsystems Inc.
09/09/10 13:36:34 INFO zookeeper.ZooKeeper: Client
environment:java.home=/usr/lib/jvm/java-6-sun-1.6.0.16/jre
09/09/10 13:36:34 INFO zookeeper.ZooKeeper: Client
environment:java.class.path=/home/stack/checkouts/workspace/svn_hbase_trunk/bin/../conf:/usr/lib/jvm/java-6-sun//lib/tools.jar:/home/stack/checkouts/workspace/svn_hbase_trunk/bin/../build/classes:/home/stack/checkouts/workspace/svn_hbase_trunk/bin/../build/test:/home/stack/checkouts/workspace/svn_hbase_trunk/bin/../build:/home/stack/checkouts/workspace/svn_hbase_trunk/bin/../lib/AgileJSON-2009-03-30.jar:/home/stack/checkouts/workspace/svn_hbase_trunk/bin/../lib/commons-cli-2.0-SNAPSHOT.jar:/home/stack/checkouts/workspace/svn_hbase_trunk/bin/../lib/commons-el-from-jetty-5.1.4.jar:/home/stack/checkouts/workspace/svn_hbase_trunk/bin/../lib/commons-httpclient-3.0.1.jar:/home/stack/checkouts/workspace/svn_hbase_trunk/bin/../lib/commons-logging-1.0.4.jar:/home/stack/checkouts/workspace/svn_hbase_trunk/bin/../lib/commons-logging-api-1.0.4.jar:/home/stack/checkouts/workspace/svn_hbase_trunk/bin/../lib/commons-math-1.1.jar:/home/stack/checkouts/workspace/svn_hbase_trunk/bin/../lib/hadoop-0.20.1-hdfs127-core.jar:/home/stack/checkouts/workspace/svn_hbase_trunk/bin/../lib/hadoop-0.20.1-test.jar:/home/stack/checkouts/workspace/svn_hbase_trunk/bin/../lib/jasper-compiler-5.5.12.jar:/home/stack/checkouts/workspace/svn_hbase_trunk/bin/../lib/jasper-runtime-5.5.12.jar:/home/stack/checkouts/workspace/svn_hbase_trunk/bin/../lib/jetty-6.1.14.jar:/home/stack/checkouts/workspace/svn_hbase_trunk/bin/../lib/jetty-util-6.1.14.jar:/home/stack/checkouts/workspace/svn_hbase_trunk/bin/../lib/jruby-complete-1.3.1.jar:/home/stack/checkouts/workspace/svn_hbase_trunk/bin/../lib/json.jar:/home/stack/checkouts/workspace/svn_hbase_trunk/bin/../lib/junit-4.7.jar:/home/stack/checkouts/workspace/svn_hbase_trunk/bin/../lib/libthrift-r771587.jar:/home/stack/checkouts/workspace/svn_hbase_trunk/bin/../lib/log4j-1.2.15.jar:/home/stack/checkouts/workspace/svn_hbase_trunk/bin/../lib/lucene-core-2.2.0.jar:/home/stack/checkouts/workspace/svn_hbase_trunk/bin/../lib/servlet-api-2.5-6.1.14.jar:/home/stack/checkouts/workspace/svn_hbase_trunk/bin/../lib/xmlenc-0.52.jar:/home/stack/checkouts/workspace/svn_hbase_trunk/bin/../lib/zookeeper-3.2.1.jar:/home/stack/checkouts/workspace/svn_hbase_trunk/bin/../lib/jsp-2.1/jsp-2.1.jar:/home/stack/checkouts/workspace/svn_hbase_trunk/bin/../lib/jsp-2.1/jsp-api-2.1.jar
09/09/10 13:36:34 INFO zookeeper.ZooKeeper: Client
environment:java.library.path=/home/stack/checkouts/workspace/svn_hbase_trunk/bin/../lib/native/Linux-amd64-64
09/09/10 13:36:34 INFO zookeeper.ZooKeeper: Client
environment:java.io.tmpdir=/tmp
09/09/10 13:36:34 INFO zookeeper.ZooKeeper: Client
environment:java.compiler=<NA>
09/09/10 13:36:34 INFO zookeeper.ZooKeeper: Client environment:os.name=Linux
09/09/10 13:36:34 INFO zookeeper.ZooKeeper: Client environment:os.arch=amd64
09/09/10 13:36:34 INFO zookeeper.ZooKeeper: Client
environment:os.version=2.6.28-11-generic
09/09/10 13:36:34 INFO zookeeper.ZooKeeper: Client environment:user.name
=stack
09/09/10 13:36:34 INFO zookeeper.ZooKeeper: Client
environment:user.home=/home/stack
09/09/10 13:36:34 INFO zookeeper.ZooKeeper: Client
environment:user.dir=/home/stack/checkouts/workspace/svn_hbase_trunk
09/09/10 13:36:34 INFO zookeeper.ZooKeeper: Initiating client connection,
connectString=localhost:2181 sessionTimeout=60000
watcher=org.apache.hadoop.hbase.client.HConnectionManager$ClientZKWatcher@5d3e754f
09/09/10 13:36:34 INFO zookeeper.ClientCnxn: zookeeper.disableAutoWatchReset
is false
09/09/10 13:36:34 INFO zookeeper.ClientCnxn: Attempting connection to server
localhost/0:0:0:0:0:0:0:1:2181
09/09/10 13:36:34 INFO zookeeper.ClientCnxn: Priming connection to
java.nio.channels.SocketChannel[connected local=/0:0:0:0:0:0:0:1:33027
remote=localhost/0:0:0:0:0:0:0:1:2181]
09/09/10 13:36:34 INFO zookeeper.ClientCnxn: Server connection successful
995 [1,2,3]
920 [1,2,3]
917 [1,2,3]
916 [1,2,3]
915 [1,2,3]
922 [1,2,3]
925 [1,2,3]
927 [1,2,3]
931 [1,2,3]
939 [1,2,3]
933 [1,2,3]
999 [1,2,3]
943 [1,2,3]
950 [1,2,3]
956 [1,2,3]
969 [1,2,3]
968 [1,2,3]
...
538 [1,2,3]
981 [1,2,3]
983 [1,2,3]
10 [1,2,3]
515 [1,2,3]
985 [1,2,3]
510 [1,2,3]
929 [1,2,3]
914 [1,2,3]
On Thu, Sep 10, 2009 at 7:06 AM, elsif <el...@gmail.com> wrote:
>
> Using a shared HBaseConfiguration has the same result. With the default
> 30 connections I would expect to see all 20 threads return a response
> instead of just the first 10.
>
> The following is an updated test case:
> --
>
> import java.io.IOException;
>
> import org.apache.hadoop.hbase.HBaseConfiguration;
> import org.apache.hadoop.hbase.KeyValue;
> import org.apache.hadoop.hbase.client.HTable;
> import org.apache.hadoop.hbase.client.Get;
> import org.apache.hadoop.hbase.client.Result;
> import org.apache.hadoop.hbase.util.Bytes;
>
> // create 'fc_test', {NAME => 'json', VERSIONS => 1}, {NAME => 'data',
> VERSIONS =>1, COMPRESSION => 'lzo' }
> // put 'fc_test', 'testing123', 'json:', "[1,2,3]"
>
> public class TableTest extends Thread {
> static HBaseConfiguration hbaseConfiguration = null;
>
> public void run() {
> try {
> String key = "testing123";
> HTable table = new HTable(hbaseConfiguration, "fc_test");
> Get get = new Get(Bytes.toBytes(key));
> get.addFamily(Bytes.toBytes("json"));
> Result result = table.get(get);
> KeyValue[] kvs = result.raw();
>
> System.out.println(Bytes.toString(kvs[0].getValue()));
> } catch (Exception e) {
> System.out.println(e.toString());
> }
>
> try {
> Thread.currentThread().sleep(1000);
> } catch (Exception e) {
> }
>
> }
>
> public static void main(String args[]) throws IOException {
> Thread t = null;
> int i;
>
> hbaseConfiguration = new HBaseConfiguration();
>
> for (i = 0; i< 20; i++) {
> t = new TableTest();
> t.start();
> }
>
> // wait for last thread
> try {
> t.join();
> } catch (InterruptedException e) {
> }
> }
> }
>
>
> stack wrote:
> > Try making the HBaseConfiguration once only (Create it in a Constructor
> or
> > pass it in from the main).
> >
> > ZooKeeper has an upper bound on the number of connections from any one IP
> > (30 by default IIRC) to protect against DoS.
> >
> > HBase HTable internally will create a new connection each time if the
> > HBaseConfiguration is new each time; otherwise, the HBase Connection
> object
> > is shared amongst HTables.
> >
> > St.Ack
> >
> >
> >
>
>
Re: Hbase 0.20.0 threading issue
Posted by elsif <el...@gmail.com>.
Using a shared HBaseConfiguration has the same result. With the default
30 connections I would expect to see all 20 threads return a response
instead of just the first 10.
The following is an updated test case:
--
import java.io.IOException;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.util.Bytes;
// create 'fc_test', {NAME => 'json', VERSIONS => 1}, {NAME => 'data',
VERSIONS =>1, COMPRESSION => 'lzo' }
// put 'fc_test', 'testing123', 'json:', "[1,2,3]"
public class TableTest extends Thread {
static HBaseConfiguration hbaseConfiguration = null;
public void run() {
try {
String key = "testing123";
HTable table = new HTable(hbaseConfiguration, "fc_test");
Get get = new Get(Bytes.toBytes(key));
get.addFamily(Bytes.toBytes("json"));
Result result = table.get(get);
KeyValue[] kvs = result.raw();
System.out.println(Bytes.toString(kvs[0].getValue()));
} catch (Exception e) {
System.out.println(e.toString());
}
try {
Thread.currentThread().sleep(1000);
} catch (Exception e) {
}
}
public static void main(String args[]) throws IOException {
Thread t = null;
int i;
hbaseConfiguration = new HBaseConfiguration();
for (i = 0; i< 20; i++) {
t = new TableTest();
t.start();
}
// wait for last thread
try {
t.join();
} catch (InterruptedException e) {
}
}
}
stack wrote:
> Try making the HBaseConfiguration once only (Create it in a Constructor or
> pass it in from the main).
>
> ZooKeeper has an upper bound on the number of connections from any one IP
> (30 by default IIRC) to protect against DoS.
>
> HBase HTable internally will create a new connection each time if the
> HBaseConfiguration is new each time; otherwise, the HBase Connection object
> is shared amongst HTables.
>
> St.Ack
>
>
>
Re: Hbase 0.20.0 threading issue
Posted by stack <st...@duboce.net>.
Try making the HBaseConfiguration once only (Create it in a Constructor or
pass it in from the main).
ZooKeeper has an upper bound on the number of connections from any one IP
(30 by default IIRC) to protect against DoS.
HBase HTable internally will create a new connection each time if the
HBaseConfiguration is new each time; otherwise, the HBase Connection object
is shared amongst HTables.
St.Ack
On Wed, Sep 9, 2009 at 6:07 PM, elsif <el...@gmail.com> wrote:
> Hello. We are trying to use Hbase in a threaded application and are
> encountering an issue with the ZooKeeper connection being lost:
>
> org.apache.hadoop.hbase.client.NoServerForRegionException: Timed out
> trying to locate root region
>
> The following sample program returns 10 results and 10 of the above
> errors. Code was compiled against the 0.20.0 svn fork and the
> pre-compiled Hadoop 0.20.0 release.
>
> import java.io.IOException;
>
> import org.apache.hadoop.hbase.HBaseConfiguration;
> import org.apache.hadoop.hbase.KeyValue;
> import org.apache.hadoop.hbase.client.HTable;
> import org.apache.hadoop.hbase.client.Get;
> import org.apache.hadoop.hbase.client.Result;
> import org.apache.hadoop.hbase.util.Bytes;
>
> // create 'fc_test', {NAME => 'json', VERSIONS => 1}, {NAME => 'data',
> VERSIONS =>1}
> // put 'fc_test', 'testing123', 'json:', "[1,2,3]"
>
> public class TableTest extends Thread {
> public void run() {
> try {
> String key = "testing123";
> HBaseConfiguration conf = new HBaseConfiguration();
> HTable table = new HTable(conf, "fc_test");
> Get get = new Get(Bytes.toBytes(key));
> get.addFamily(Bytes.toBytes("json"));
> Result result = table.get(get);
> KeyValue[] kvs = result.raw();
>
> System.out.println(Bytes.toString(kvs[0].getValue()));
> } catch (Exception e) {
> System.out.println(e.toString());
> }
>
> try {
> Thread.currentThread().sleep(1000);
> } catch (Exception e) {
> }
>
> }
>
> public static void main(String args[]) throws IOException {
> Thread t = null;
> int i;
>
> for (i = 0; i< 20; i++) {
> t = new TableTest();
> t.start();
> }
>
> // wait for last thread
> try {
> t.join();
> } catch (InterruptedException e) {
> }
> }
> }
>
>
>
> #!/bin/bash
>
> export
> CLASSPATH=.:/opt/hbase/conf:/opt/hbase/build/hbase-0.20.0.jar:/opt/hbase/lib/AgileJSON-2009-03-30.jar:/opt/hbase/lib/commons-cli-2.0-SNAPSHOT.jar:/opt/hbase/lib/commons-el-from-jetty-5.1.4.jar:/opt/hbase/lib/commons-httpclient-3.0.1.jar:/opt/hbase/lib/commons-logging-1.0.4.jar:/opt/hbase/lib/commons-logging-api-1.0.4.jar:/opt/hbase/lib/commons-math-1.1.jar:/opt/hbase/lib/hadoop-0.20.0-plus4681-core.jar:/opt/hbase/lib/hadoop-0.20.0-test.jar:/opt/hbase/lib/hadoop-gpl-compression-0.1.0-dev.jar:/opt/hbase/lib/jasper-compiler-5.5.12.jar:/opt/hbase/lib/jasper-runtime-5.5.12.jar:/opt/hbase/lib/jetty-6.1.14.jar:/opt/hbase/lib/jetty-util-6.1.14.jar:/opt/hbase/lib/jruby-complete-1.2.0.jar:/opt/hbase/lib/json.jar:/opt/hbase/lib/junit-3.8.1.jar:/opt/hbase/lib/libthrift-r771587.jar:/opt/hbase/lib/log4j-1.2.15.jar:/opt/hbase/lib/lucene-core-2.2.0.jar:/opt/hbase/lib/servlet-api-2.5-6.1.14.jar:/opt/hbase/lib/xmlenc-0.52.jar:/opt/hbase/lib/zookeeper-r785019-hbase-1329.jar
>
> javac -classpath ${CLASSPATH} TableTest.java
>
> java -classpath ${CLASSPATH} TableTest 2>err.log
>
>