You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@hbase.apache.org by Oleg Ruchovets <or...@gmail.com> on 2011/03/19 22:19:36 UTC
hbase heap size
Hi , we started our tests on cluster ( hbase 0.90.1 , hadoop append) ,
I set HBASE_HEAPSIZE to 4000m in hbase-env.sh and got 3 processes which
has heap size 4000m:
my questions are:
1)What is the way to set separately heap size for these processes. In case
I want to give to zookeper less heap size what is the properties should I
use?
2)Assuming that I want to set 4000m for hbase, Does it mean that all
these 3 processes have to have 4000m heap size? What is the advisable heap
size for hbase?
*the processes are: *
hadoop 10291 1 0 Mar17 ? 00:00:18 /usr/java/default//bin/java
-Xmx4000m -XX:+HeapDumpOnOutOfMemoryError -XX:+UseConcMarkSweepGC
-XX:+CMSIncrementalMode -verbose:gc -XX:+PrintGCDetails
-XX:+PrintGCTimeStamps
-Xloggc:/opt/hbase/bin/../logs/gc/gc-hbase_10291_20110317_191315.log
-Dhbase.log.dir=/opt/hbase/bin/../logs
-Dhbase.log.file=hbase-hadoop-zookeeper-hadoop01.infolinks.local.log
-Dhbase.home.dir=/opt/hbase/bin/.. -Dhbase.id.str=hadoop
-Dhbase.root.logger=INFO,DRFA -classpath
/opt/hbase/bin/../conf:/usr/java/default//lib/tools.jar:/opt/hbase/bin/..:/opt/hbase/bin/../hbase-0.90.1.jar:/opt/hbase/bin/../hbase-0.90.1-tests.jar:/opt/hbase/bin/../lib/avro-1.3.3.jar:/opt/hbase/bin/../lib/commons-cli-1.2.jar:/opt/hbase/bin/../lib/commons-codec-1.4.jar:/opt/hbase/bin/../lib/commons-el-1.0.jar:/opt/hbase/bin/../lib/commons-httpclient-3.1.jar:/opt/hbase/bin/../lib/commons-lang-2.5.jar:/opt/hbase/bin/../lib/commons-logging-1.1.1.jar:/opt/hbase/bin/../lib/guava-r06.jar:/opt/hbase/bin/../lib/hadoop-core-0.20.3-SNAPSHOT.jar:/opt/hbase/bin/../lib/jasper-compiler-5.5.23.jar:/opt/hbase/bin/../lib/jasper-runtime-5.5.23.jar:/opt/hbase/bin/../lib/jaxb-api-2.1.jar:/opt/hbase/bin/../lib/jersey-core-1.4.jar:/opt/hbase/bin/../lib/jersey-json-1.4.jar:/opt/hbase/bin/../lib/jersey-server-1.4.jar:/opt/hbase/bin/../lib/jetty-6.1.26.jar:/opt/hbase/bin/../lib/jetty-util-6.1.26.jar:/opt/hbase/bin/../lib/jruby-complete-1.0.3.jar:/opt/hbase/bin/../lib/jsp-2.1-6.1.14.jar:/opt/hbase/bin/../lib/jsp-api-2.1-6.1.14.jar:/opt/hbase/bin/../lib/jsr311-api-1.1.1.jar:/opt/hbase/bin/../lib/log4j-1.2.16.jar:/opt/hbase/bin/../lib/protobuf-java-2.3.0.jar:/opt/hbase/bin/../lib/servlet-api-2.5-6.1.14.jar:/opt/hbase/bin/../lib/slf4j-api-1.5.8.jar:/opt/hbase/bin/../lib/slf4j-log4j12-1.5.8.jar:/opt/hbase/bin/../lib/stax-api-1.0.1.jar:/opt/hbase/bin/../lib/thrift-0.2.0.jar:/opt/hbase/bin/../lib/zookeeper-3.3.2.jar
org.apache.hadoop.hbase.zookeeper.HQuorumPeer start
hadoop 10368 1 0 Mar17 pts/2 00:05:50 /usr/java/default//bin/java
-Xmx4000m -XX:+HeapDumpOnOutOfMemoryError -XX:+UseConcMarkSweepGC
-XX:+CMSIncrementalMode -verbose:gc -XX:+PrintGCDetails
-XX:+PrintGCTimeStamps
-Xloggc:/opt/hbase/bin/../logs/gc/gc-hbase_10368_20110317_191316.log
-Dhbase.log.dir=/opt/hbase/bin/../logs
-Dhbase.log.file=hbase-hadoop-master-hadoop01.infolinks.local.log
-Dhbase.home.dir=/opt/hbase/bin/.. -Dhbase.id.str=hadoop
-Dhbase.root.logger=INFO,DRFA -classpath
/opt/hbase/bin/../conf:/usr/java/default//lib/tools.jar:/opt/hbase/bin/..:/opt/hbase/bin/../hbase-0.90.1.jar:/opt/hbase/bin/../hbase-0.90.1-tests.jar:/opt/hbase/bin/../lib/avro-1.3.3.jar:/opt/hbase/bin/../lib/commons-cli-1.2.jar:/opt/hbase/bin/../lib/commons-codec-1.4.jar:/opt/hbase/bin/../lib/commons-el-1.0.jar:/opt/hbase/bin/../lib/commons-httpclient-3.1.jar:/opt/hbase/bin/../lib/commons-lang-2.5.jar:/opt/hbase/bin/../lib/commons-logging-1.1.1.jar:/opt/hbase/bin/../lib/guava-r06.jar:/opt/hbase/bin/../lib/hadoop-core-0.20.3-SNAPSHOT.jar:/opt/hbase/bin/../lib/jasper-compiler-5.5.23.jar:/opt/hbase/bin/../lib/jasper-runtime-5.5.23.jar:/opt/hbase/bin/../lib/jaxb-api-2.1.jar:/opt/hbase/bin/../lib/jersey-core-1.4.jar:/opt/hbase/bin/../lib/jersey-json-1.4.jar:/opt/hbase/bin/../lib/jersey-server-1.4.jar:/opt/hbase/bin/../lib/jetty-6.1.26.jar:/opt/hbase/bin/../lib/jetty-util-6.1.26.jar:/opt/hbase/bin/../lib/jruby-complete-1.0.3.jar:/opt/hbase/bin/../lib/jsp-2.1-6.1.14.jar:/opt/hbase/bin/../lib/jsp-api-2.1-6.1.14.jar:/opt/hbase/bin/../lib/jsr311-api-1.1.1.jar:/opt/hbase/bin/../lib/log4j-1.2.16.jar:/opt/hbase/bin/../lib/protobuf-java-2.3.0.jar:/opt/hbase/bin/../lib/servlet-api-2.5-6.1.14.jar:/opt/hbase/bin/../lib/slf4j-api-1.5.8.jar:/opt/hbase/bin/../lib/slf4j-log4j12-1.5.8.jar:/opt/hbase/bin/../lib/stax-api-1.0.1.jar:/opt/hbase/bin/../lib/thrift-0.2.0.jar:/opt/hbase/bin/../lib/zookeeper-3.3.2.jar
org.apache.hadoop.hbase.master.HMaster start
hadoop 10548 1 5 Mar17 ? 02:39:01 /usr/java/default//bin/java
-Xmx4000m -XX:+HeapDumpOnOutOfMemoryError -XX:+UseConcMarkSweepGC
-XX:+CMSIncrementalMode -verbose:gc -XX:+PrintGCDetails
-XX:+PrintGCTimeStamps
-Xloggc:/opt/hbase/bin/../logs/gc/gc-hbase_10548_20110317_191318.log
-Dhbase.log.dir=/opt/hbase/bin/../logs
-Dhbase.log.file=hbase-hadoop-regionserver-hadoop01.infolinks.local.log
-Dhbase.home.dir=/opt/hbase/bin/.. -Dhbase.id.str=hadoop
-Dhbase.root.logger=INFO,DRFA -classpath
/opt/hbase/bin/../conf:/usr/java/default//lib/tools.jar:/opt/hbase/bin/..:/opt/hbase/bin/../hbase-0.90.1.jar:/opt/hbase/bin/../hbase-0.90.1-tests.jar:/opt/hbase/bin/../lib/avro-1.3.3.jar:/opt/hbase/bin/../lib/commons-cli-1.2.jar:/opt/hbase/bin/../lib/commons-codec-1.4.jar:/opt/hbase/bin/../lib/commons-el-1.0.jar:/opt/hbase/bin/../lib/commons-httpclient-3.1.jar:/opt/hbase/bin/../lib/commons-lang-2.5.jar:/opt/hbase/bin/../lib/commons-logging-1.1.1.jar:/opt/hbase/bin/../lib/guava-r06.jar:/opt/hbase/bin/../lib/hadoop-core-0.20.3-SNAPSHOT.jar:/opt/hbase/bin/../lib/jasper-compiler-5.5.23.jar:/opt/hbase/bin/../lib/jasper-runtime-5.5.23.jar:/opt/hbase/bin/../lib/jaxb-api-2.1.jar:/opt/hbase/bin/../lib/jersey-core-1.4.jar:/opt/hbase/bin/../lib/jersey-json-1.4.jar:/opt/hbase/bin/../lib/jersey-server-1.4.jar:/opt/hbase/bin/../lib/jetty-6.1.26.jar:/opt/hbase/bin/../lib/jetty-util-6.1.26.jar:/opt/hbase/bin/../lib/jruby-complete-1.0.3.jar:/opt/hbase/bin/../lib/jsp-2.1-6.1.14.jar:/opt/hbase/bin/../lib/jsp-api-2.1-6.1.14.jar:/opt/hbase/bin/../lib/jsr311-api-1.1.1.jar:/opt/hbase/bin/../lib/log4j-1.2.16.jar:/opt/hbase/bin/../lib/protobuf-java-2.3.0.jar:/opt/hbase/bin/../lib/servlet-api-2.5-6.1.14.jar:/opt/hbase/bin/../lib/slf4j-api-1.5.8.jar:/opt/hbase/bin/../lib/slf4j-log4j12-1.5.8.jar:/opt/hbase/bin/../lib/stax-api-1.0.1.jar:/opt/hbase/bin/../lib/thrift-0.2.0.jar:/opt/hbase/bin/../lib/zookeeper-3.3.2.jar
org.apache.hadoop.hbase.regionserver.HRegionServer start
Thanks in advance
Oleg.
Re: hbase heap size
Posted by Harsh J <qw...@gmail.com>.
On Sun, Mar 20, 2011 at 2:58 AM, Oleg Ruchovets <or...@gmail.com> wrote:
> Thank you St.Ack
> the question is regarding setting heap size for hbase:
>
> As I understand there are 3 processes HBASE master , Hbase Region server ,
> Zookeper.
> What is the heap size should I set for these processes? I don't remember
> where do I see 4000m was recommended , but does it mean that all 3 processes
> have to have 4000m?
Perhaps you saw this in
http://www.slideshare.net/schubertzhang/hbase-0200-performance-evaluation?
ZooKeeper is managed external of HBase in production. It should not
really require a ~4G heap space.
RegionServer should need all the extra RAM it can get, if am right, so
set it for those surely (more than 4 is good too, if available).
--
Harsh J
http://harshj.com
Re: hbase heap size
Posted by Suraj Varma <sv...@gmail.com>.
Oleg:
Instead of setting the heap size using the common HBASE_HEAP_SIZE, use
the process specific OPTS to set it.
As Stack says, for instance to set zookeeper specific heap size, you
can uncomment and set the heap size
export HBASE_ZOOKEEPER_OPTS="-Xmx1000m $HBASE_JMX_BASE
-Dcom.sun.management.jmxremote.port=10104"
and so forth instead of HBASE_HEAP_SIZE.
--Suraj
On Sat, Mar 19, 2011 at 2:28 PM, Oleg Ruchovets <or...@gmail.com> wrote:
> Thank you St.Ack
> the question is regarding setting heap size for hbase:
>
> As I understand there are 3 processes HBASE master , Hbase Region server ,
> Zookeper.
> What is the heap size should I set for these processes? I don't remember
> where do I see 4000m was recommended , but does it mean that all 3 processes
> have to have 4000m?
>
>
> On Sat, Mar 19, 2011 at 11:21 PM, Stack <st...@duboce.net> wrote:
>
>> See this section in your hbase-env.sh:
>>
>> # export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false
>> -Dcom.sun.management.jmxremote.authenticate=false"
>> # export HBASE_MASTER_OPTS="$HBASE_JMX_BASE
>> -Dcom.sun.management.jmxremote.port=10101
>> -javaagent:lib/HelloWorldAgent.jar"
>> # export HBASE_REGIONSERVER_OPTS="$HBASE_JMX_BASE
>> -Dcom.sun.management.jmxremote.port=10102"
>> # export HBASE_THRIFT_OPTS="$HBASE_JMX_BASE
>> -Dcom.sun.management.jmxremote.port=10103"
>> # export HBASE_ZOOKEEPER_OPTS="$HBASE_JMX_BASE
>> -Dcom.sun.management.jmxremote.port=10104"
>>
>> Uncomment as appropriate.
>>
>> St.Ack
>>
>> On Sat, Mar 19, 2011 at 2:19 PM, Oleg Ruchovets <or...@gmail.com>
>> wrote:
>> > Hi , we started our tests on cluster ( hbase 0.90.1 , hadoop append) ,
>> > I set HBASE_HEAPSIZE to 4000m in hbase-env.sh and got 3 processes
>> which
>> > has heap size 4000m:
>> >
>> > my questions are:
>> > 1)What is the way to set separately heap size for these processes. In
>> case
>> > I want to give to zookeper less heap size what is the properties should I
>> > use?
>> > 2)Assuming that I want to set 4000m for hbase, Does it mean that all
>> > these 3 processes have to have 4000m heap size? What is the advisable
>> heap
>> > size for hbase?
>> >
>> > *the processes are: *
>> >
>> >
>> >
>> > hadoop 10291 1 0 Mar17 ? 00:00:18
>> /usr/java/default//bin/java
>> > -Xmx4000m -XX:+HeapDumpOnOutOfMemoryError -XX:+UseConcMarkSweepGC
>> > -XX:+CMSIncrementalMode -verbose:gc -XX:+PrintGCDetails
>> > -XX:+PrintGCTimeStamps
>> > -Xloggc:/opt/hbase/bin/../logs/gc/gc-hbase_10291_20110317_191315.log
>> > -Dhbase.log.dir=/opt/hbase/bin/../logs
>> > -Dhbase.log.file=hbase-hadoop-zookeeper-hadoop01.infolinks.local.log
>> > -Dhbase.home.dir=/opt/hbase/bin/.. -Dhbase.id.str=hadoop
>> > -Dhbase.root.logger=INFO,DRFA -classpath
>> >
>> /opt/hbase/bin/../conf:/usr/java/default//lib/tools.jar:/opt/hbase/bin/..:/opt/hbase/bin/../hbase-0.90.1.jar:/opt/hbase/bin/../hbase-0.90.1-tests.jar:/opt/hbase/bin/../lib/avro-1.3.3.jar:/opt/hbase/bin/../lib/commons-cli-1.2.jar:/opt/hbase/bin/../lib/commons-codec-1.4.jar:/opt/hbase/bin/../lib/commons-el-1.0.jar:/opt/hbase/bin/../lib/commons-httpclient-3.1.jar:/opt/hbase/bin/../lib/commons-lang-2.5.jar:/opt/hbase/bin/../lib/commons-logging-1.1.1.jar:/opt/hbase/bin/../lib/guava-r06.jar:/opt/hbase/bin/../lib/hadoop-core-0.20.3-SNAPSHOT.jar:/opt/hbase/bin/../lib/jasper-compiler-5.5.23.jar:/opt/hbase/bin/../lib/jasper-runtime-5.5.23.jar:/opt/hbase/bin/../lib/jaxb-api-2.1.jar:/opt/hbase/bin/../lib/jersey-core-1.4.jar:/opt/hbase/bin/../lib/jersey-json-1.4.jar:/opt/hbase/bin/../lib/jersey-server-1.4.jar:/opt/hbase/bin/../lib/jetty-6.1.26.jar:/opt/hbase/bin/../lib/jetty-util-6.1.26.jar:/opt/hbase/bin/../lib/jruby-complete-1.0.3.jar:/opt/hbase/bin/../lib/jsp-2.1-6.1.14.jar:/opt/hbase/bin/../lib/jsp-api-2.1-6.1.14.jar:/opt/hbase/bin/../lib/jsr311-api-1.1.1.jar:/opt/hbase/bin/../lib/log4j-1.2.16.jar:/opt/hbase/bin/../lib/protobuf-java-2.3.0.jar:/opt/hbase/bin/../lib/servlet-api-2.5-6.1.14.jar:/opt/hbase/bin/../lib/slf4j-api-1.5.8.jar:/opt/hbase/bin/../lib/slf4j-log4j12-1.5.8.jar:/opt/hbase/bin/../lib/stax-api-1.0.1.jar:/opt/hbase/bin/../lib/thrift-0.2.0.jar:/opt/hbase/bin/../lib/zookeeper-3.3.2.jar
>> > org.apache.hadoop.hbase.zookeeper.HQuorumPeer start
>> > hadoop 10368 1 0 Mar17 pts/2 00:05:50
>> /usr/java/default//bin/java
>> > -Xmx4000m -XX:+HeapDumpOnOutOfMemoryError -XX:+UseConcMarkSweepGC
>> > -XX:+CMSIncrementalMode -verbose:gc -XX:+PrintGCDetails
>> > -XX:+PrintGCTimeStamps
>> > -Xloggc:/opt/hbase/bin/../logs/gc/gc-hbase_10368_20110317_191316.log
>> > -Dhbase.log.dir=/opt/hbase/bin/../logs
>> > -Dhbase.log.file=hbase-hadoop-master-hadoop01.infolinks.local.log
>> > -Dhbase.home.dir=/opt/hbase/bin/.. -Dhbase.id.str=hadoop
>> > -Dhbase.root.logger=INFO,DRFA -classpath
>> >
>> /opt/hbase/bin/../conf:/usr/java/default//lib/tools.jar:/opt/hbase/bin/..:/opt/hbase/bin/../hbase-0.90.1.jar:/opt/hbase/bin/../hbase-0.90.1-tests.jar:/opt/hbase/bin/../lib/avro-1.3.3.jar:/opt/hbase/bin/../lib/commons-cli-1.2.jar:/opt/hbase/bin/../lib/commons-codec-1.4.jar:/opt/hbase/bin/../lib/commons-el-1.0.jar:/opt/hbase/bin/../lib/commons-httpclient-3.1.jar:/opt/hbase/bin/../lib/commons-lang-2.5.jar:/opt/hbase/bin/../lib/commons-logging-1.1.1.jar:/opt/hbase/bin/../lib/guava-r06.jar:/opt/hbase/bin/../lib/hadoop-core-0.20.3-SNAPSHOT.jar:/opt/hbase/bin/../lib/jasper-compiler-5.5.23.jar:/opt/hbase/bin/../lib/jasper-runtime-5.5.23.jar:/opt/hbase/bin/../lib/jaxb-api-2.1.jar:/opt/hbase/bin/../lib/jersey-core-1.4.jar:/opt/hbase/bin/../lib/jersey-json-1.4.jar:/opt/hbase/bin/../lib/jersey-server-1.4.jar:/opt/hbase/bin/../lib/jetty-6.1.26.jar:/opt/hbase/bin/../lib/jetty-util-6.1.26.jar:/opt/hbase/bin/../lib/jruby-complete-1.0.3.jar:/opt/hbase/bin/../lib/jsp-2.1-6.1.14.jar:/opt/hbase/bin/../lib/jsp-api-2.1-6.1.14.jar:/opt/hbase/bin/../lib/jsr311-api-1.1.1.jar:/opt/hbase/bin/../lib/log4j-1.2.16.jar:/opt/hbase/bin/../lib/protobuf-java-2.3.0.jar:/opt/hbase/bin/../lib/servlet-api-2.5-6.1.14.jar:/opt/hbase/bin/../lib/slf4j-api-1.5.8.jar:/opt/hbase/bin/../lib/slf4j-log4j12-1.5.8.jar:/opt/hbase/bin/../lib/stax-api-1.0.1.jar:/opt/hbase/bin/../lib/thrift-0.2.0.jar:/opt/hbase/bin/../lib/zookeeper-3.3.2.jar
>> > org.apache.hadoop.hbase.master.HMaster start
>> > hadoop 10548 1 5 Mar17 ? 02:39:01
>> /usr/java/default//bin/java
>> > -Xmx4000m -XX:+HeapDumpOnOutOfMemoryError -XX:+UseConcMarkSweepGC
>> > -XX:+CMSIncrementalMode -verbose:gc -XX:+PrintGCDetails
>> > -XX:+PrintGCTimeStamps
>> > -Xloggc:/opt/hbase/bin/../logs/gc/gc-hbase_10548_20110317_191318.log
>> > -Dhbase.log.dir=/opt/hbase/bin/../logs
>> > -Dhbase.log.file=hbase-hadoop-regionserver-hadoop01.infolinks.local.log
>> > -Dhbase.home.dir=/opt/hbase/bin/.. -Dhbase.id.str=hadoop
>> > -Dhbase.root.logger=INFO,DRFA -classpath
>> >
>> /opt/hbase/bin/../conf:/usr/java/default//lib/tools.jar:/opt/hbase/bin/..:/opt/hbase/bin/../hbase-0.90.1.jar:/opt/hbase/bin/../hbase-0.90.1-tests.jar:/opt/hbase/bin/../lib/avro-1.3.3.jar:/opt/hbase/bin/../lib/commons-cli-1.2.jar:/opt/hbase/bin/../lib/commons-codec-1.4.jar:/opt/hbase/bin/../lib/commons-el-1.0.jar:/opt/hbase/bin/../lib/commons-httpclient-3.1.jar:/opt/hbase/bin/../lib/commons-lang-2.5.jar:/opt/hbase/bin/../lib/commons-logging-1.1.1.jar:/opt/hbase/bin/../lib/guava-r06.jar:/opt/hbase/bin/../lib/hadoop-core-0.20.3-SNAPSHOT.jar:/opt/hbase/bin/../lib/jasper-compiler-5.5.23.jar:/opt/hbase/bin/../lib/jasper-runtime-5.5.23.jar:/opt/hbase/bin/../lib/jaxb-api-2.1.jar:/opt/hbase/bin/../lib/jersey-core-1.4.jar:/opt/hbase/bin/../lib/jersey-json-1.4.jar:/opt/hbase/bin/../lib/jersey-server-1.4.jar:/opt/hbase/bin/../lib/jetty-6.1.26.jar:/opt/hbase/bin/../lib/jetty-util-6.1.26.jar:/opt/hbase/bin/../lib/jruby-complete-1.0.3.jar:/opt/hbase/bin/../lib/jsp-2.1-6.1.14.jar:/opt/hbase/bin/../lib/jsp-api-2.1-6.1.14.jar:/opt/hbase/bin/../lib/jsr311-api-1.1.1.jar:/opt/hbase/bin/../lib/log4j-1.2.16.jar:/opt/hbase/bin/../lib/protobuf-java-2.3.0.jar:/opt/hbase/bin/../lib/servlet-api-2.5-6.1.14.jar:/opt/hbase/bin/../lib/slf4j-api-1.5.8.jar:/opt/hbase/bin/../lib/slf4j-log4j12-1.5.8.jar:/opt/hbase/bin/../lib/stax-api-1.0.1.jar:/opt/hbase/bin/../lib/thrift-0.2.0.jar:/opt/hbase/bin/../lib/zookeeper-3.3.2.jar
>> > org.apache.hadoop.hbase.regionserver.HRegionServer start
>> >
>> > Thanks in advance
>> > Oleg.
>> >
>>
>
Re: hbase heap size
Posted by Oleg Ruchovets <or...@gmail.com>.
Thank you St.Ack
the question is regarding setting heap size for hbase:
As I understand there are 3 processes HBASE master , Hbase Region server ,
Zookeper.
What is the heap size should I set for these processes? I don't remember
where do I see 4000m was recommended , but does it mean that all 3 processes
have to have 4000m?
On Sat, Mar 19, 2011 at 11:21 PM, Stack <st...@duboce.net> wrote:
> See this section in your hbase-env.sh:
>
> # export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false
> -Dcom.sun.management.jmxremote.authenticate=false"
> # export HBASE_MASTER_OPTS="$HBASE_JMX_BASE
> -Dcom.sun.management.jmxremote.port=10101
> -javaagent:lib/HelloWorldAgent.jar"
> # export HBASE_REGIONSERVER_OPTS="$HBASE_JMX_BASE
> -Dcom.sun.management.jmxremote.port=10102"
> # export HBASE_THRIFT_OPTS="$HBASE_JMX_BASE
> -Dcom.sun.management.jmxremote.port=10103"
> # export HBASE_ZOOKEEPER_OPTS="$HBASE_JMX_BASE
> -Dcom.sun.management.jmxremote.port=10104"
>
> Uncomment as appropriate.
>
> St.Ack
>
> On Sat, Mar 19, 2011 at 2:19 PM, Oleg Ruchovets <or...@gmail.com>
> wrote:
> > Hi , we started our tests on cluster ( hbase 0.90.1 , hadoop append) ,
> > I set HBASE_HEAPSIZE to 4000m in hbase-env.sh and got 3 processes
> which
> > has heap size 4000m:
> >
> > my questions are:
> > 1)What is the way to set separately heap size for these processes. In
> case
> > I want to give to zookeper less heap size what is the properties should I
> > use?
> > 2)Assuming that I want to set 4000m for hbase, Does it mean that all
> > these 3 processes have to have 4000m heap size? What is the advisable
> heap
> > size for hbase?
> >
> > *the processes are: *
> >
> >
> >
> > hadoop 10291 1 0 Mar17 ? 00:00:18
> /usr/java/default//bin/java
> > -Xmx4000m -XX:+HeapDumpOnOutOfMemoryError -XX:+UseConcMarkSweepGC
> > -XX:+CMSIncrementalMode -verbose:gc -XX:+PrintGCDetails
> > -XX:+PrintGCTimeStamps
> > -Xloggc:/opt/hbase/bin/../logs/gc/gc-hbase_10291_20110317_191315.log
> > -Dhbase.log.dir=/opt/hbase/bin/../logs
> > -Dhbase.log.file=hbase-hadoop-zookeeper-hadoop01.infolinks.local.log
> > -Dhbase.home.dir=/opt/hbase/bin/.. -Dhbase.id.str=hadoop
> > -Dhbase.root.logger=INFO,DRFA -classpath
> >
> /opt/hbase/bin/../conf:/usr/java/default//lib/tools.jar:/opt/hbase/bin/..:/opt/hbase/bin/../hbase-0.90.1.jar:/opt/hbase/bin/../hbase-0.90.1-tests.jar:/opt/hbase/bin/../lib/avro-1.3.3.jar:/opt/hbase/bin/../lib/commons-cli-1.2.jar:/opt/hbase/bin/../lib/commons-codec-1.4.jar:/opt/hbase/bin/../lib/commons-el-1.0.jar:/opt/hbase/bin/../lib/commons-httpclient-3.1.jar:/opt/hbase/bin/../lib/commons-lang-2.5.jar:/opt/hbase/bin/../lib/commons-logging-1.1.1.jar:/opt/hbase/bin/../lib/guava-r06.jar:/opt/hbase/bin/../lib/hadoop-core-0.20.3-SNAPSHOT.jar:/opt/hbase/bin/../lib/jasper-compiler-5.5.23.jar:/opt/hbase/bin/../lib/jasper-runtime-5.5.23.jar:/opt/hbase/bin/../lib/jaxb-api-2.1.jar:/opt/hbase/bin/../lib/jersey-core-1.4.jar:/opt/hbase/bin/../lib/jersey-json-1.4.jar:/opt/hbase/bin/../lib/jersey-server-1.4.jar:/opt/hbase/bin/../lib/jetty-6.1.26.jar:/opt/hbase/bin/../lib/jetty-util-6.1.26.jar:/opt/hbase/bin/../lib/jruby-complete-1.0.3.jar:/opt/hbase/bin/../lib/jsp-2.1-6.1.14.jar:/opt/hbase/bin/../lib/jsp-api-2.1-6.1.14.jar:/opt/hbase/bin/../lib/jsr311-api-1.1.1.jar:/opt/hbase/bin/../lib/log4j-1.2.16.jar:/opt/hbase/bin/../lib/protobuf-java-2.3.0.jar:/opt/hbase/bin/../lib/servlet-api-2.5-6.1.14.jar:/opt/hbase/bin/../lib/slf4j-api-1.5.8.jar:/opt/hbase/bin/../lib/slf4j-log4j12-1.5.8.jar:/opt/hbase/bin/../lib/stax-api-1.0.1.jar:/opt/hbase/bin/../lib/thrift-0.2.0.jar:/opt/hbase/bin/../lib/zookeeper-3.3.2.jar
> > org.apache.hadoop.hbase.zookeeper.HQuorumPeer start
> > hadoop 10368 1 0 Mar17 pts/2 00:05:50
> /usr/java/default//bin/java
> > -Xmx4000m -XX:+HeapDumpOnOutOfMemoryError -XX:+UseConcMarkSweepGC
> > -XX:+CMSIncrementalMode -verbose:gc -XX:+PrintGCDetails
> > -XX:+PrintGCTimeStamps
> > -Xloggc:/opt/hbase/bin/../logs/gc/gc-hbase_10368_20110317_191316.log
> > -Dhbase.log.dir=/opt/hbase/bin/../logs
> > -Dhbase.log.file=hbase-hadoop-master-hadoop01.infolinks.local.log
> > -Dhbase.home.dir=/opt/hbase/bin/.. -Dhbase.id.str=hadoop
> > -Dhbase.root.logger=INFO,DRFA -classpath
> >
> /opt/hbase/bin/../conf:/usr/java/default//lib/tools.jar:/opt/hbase/bin/..:/opt/hbase/bin/../hbase-0.90.1.jar:/opt/hbase/bin/../hbase-0.90.1-tests.jar:/opt/hbase/bin/../lib/avro-1.3.3.jar:/opt/hbase/bin/../lib/commons-cli-1.2.jar:/opt/hbase/bin/../lib/commons-codec-1.4.jar:/opt/hbase/bin/../lib/commons-el-1.0.jar:/opt/hbase/bin/../lib/commons-httpclient-3.1.jar:/opt/hbase/bin/../lib/commons-lang-2.5.jar:/opt/hbase/bin/../lib/commons-logging-1.1.1.jar:/opt/hbase/bin/../lib/guava-r06.jar:/opt/hbase/bin/../lib/hadoop-core-0.20.3-SNAPSHOT.jar:/opt/hbase/bin/../lib/jasper-compiler-5.5.23.jar:/opt/hbase/bin/../lib/jasper-runtime-5.5.23.jar:/opt/hbase/bin/../lib/jaxb-api-2.1.jar:/opt/hbase/bin/../lib/jersey-core-1.4.jar:/opt/hbase/bin/../lib/jersey-json-1.4.jar:/opt/hbase/bin/../lib/jersey-server-1.4.jar:/opt/hbase/bin/../lib/jetty-6.1.26.jar:/opt/hbase/bin/../lib/jetty-util-6.1.26.jar:/opt/hbase/bin/../lib/jruby-complete-1.0.3.jar:/opt/hbase/bin/../lib/jsp-2.1-6.1.14.jar:/opt/hbase/bin/../lib/jsp-api-2.1-6.1.14.jar:/opt/hbase/bin/../lib/jsr311-api-1.1.1.jar:/opt/hbase/bin/../lib/log4j-1.2.16.jar:/opt/hbase/bin/../lib/protobuf-java-2.3.0.jar:/opt/hbase/bin/../lib/servlet-api-2.5-6.1.14.jar:/opt/hbase/bin/../lib/slf4j-api-1.5.8.jar:/opt/hbase/bin/../lib/slf4j-log4j12-1.5.8.jar:/opt/hbase/bin/../lib/stax-api-1.0.1.jar:/opt/hbase/bin/../lib/thrift-0.2.0.jar:/opt/hbase/bin/../lib/zookeeper-3.3.2.jar
> > org.apache.hadoop.hbase.master.HMaster start
> > hadoop 10548 1 5 Mar17 ? 02:39:01
> /usr/java/default//bin/java
> > -Xmx4000m -XX:+HeapDumpOnOutOfMemoryError -XX:+UseConcMarkSweepGC
> > -XX:+CMSIncrementalMode -verbose:gc -XX:+PrintGCDetails
> > -XX:+PrintGCTimeStamps
> > -Xloggc:/opt/hbase/bin/../logs/gc/gc-hbase_10548_20110317_191318.log
> > -Dhbase.log.dir=/opt/hbase/bin/../logs
> > -Dhbase.log.file=hbase-hadoop-regionserver-hadoop01.infolinks.local.log
> > -Dhbase.home.dir=/opt/hbase/bin/.. -Dhbase.id.str=hadoop
> > -Dhbase.root.logger=INFO,DRFA -classpath
> >
> /opt/hbase/bin/../conf:/usr/java/default//lib/tools.jar:/opt/hbase/bin/..:/opt/hbase/bin/../hbase-0.90.1.jar:/opt/hbase/bin/../hbase-0.90.1-tests.jar:/opt/hbase/bin/../lib/avro-1.3.3.jar:/opt/hbase/bin/../lib/commons-cli-1.2.jar:/opt/hbase/bin/../lib/commons-codec-1.4.jar:/opt/hbase/bin/../lib/commons-el-1.0.jar:/opt/hbase/bin/../lib/commons-httpclient-3.1.jar:/opt/hbase/bin/../lib/commons-lang-2.5.jar:/opt/hbase/bin/../lib/commons-logging-1.1.1.jar:/opt/hbase/bin/../lib/guava-r06.jar:/opt/hbase/bin/../lib/hadoop-core-0.20.3-SNAPSHOT.jar:/opt/hbase/bin/../lib/jasper-compiler-5.5.23.jar:/opt/hbase/bin/../lib/jasper-runtime-5.5.23.jar:/opt/hbase/bin/../lib/jaxb-api-2.1.jar:/opt/hbase/bin/../lib/jersey-core-1.4.jar:/opt/hbase/bin/../lib/jersey-json-1.4.jar:/opt/hbase/bin/../lib/jersey-server-1.4.jar:/opt/hbase/bin/../lib/jetty-6.1.26.jar:/opt/hbase/bin/../lib/jetty-util-6.1.26.jar:/opt/hbase/bin/../lib/jruby-complete-1.0.3.jar:/opt/hbase/bin/../lib/jsp-2.1-6.1.14.jar:/opt/hbase/bin/../lib/jsp-api-2.1-6.1.14.jar:/opt/hbase/bin/../lib/jsr311-api-1.1.1.jar:/opt/hbase/bin/../lib/log4j-1.2.16.jar:/opt/hbase/bin/../lib/protobuf-java-2.3.0.jar:/opt/hbase/bin/../lib/servlet-api-2.5-6.1.14.jar:/opt/hbase/bin/../lib/slf4j-api-1.5.8.jar:/opt/hbase/bin/../lib/slf4j-log4j12-1.5.8.jar:/opt/hbase/bin/../lib/stax-api-1.0.1.jar:/opt/hbase/bin/../lib/thrift-0.2.0.jar:/opt/hbase/bin/../lib/zookeeper-3.3.2.jar
> > org.apache.hadoop.hbase.regionserver.HRegionServer start
> >
> > Thanks in advance
> > Oleg.
> >
>
Re: hbase heap size
Posted by Stack <st...@duboce.net>.
See this section in your hbase-env.sh:
# export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false
-Dcom.sun.management.jmxremote.authenticate=false"
# export HBASE_MASTER_OPTS="$HBASE_JMX_BASE
-Dcom.sun.management.jmxremote.port=10101
-javaagent:lib/HelloWorldAgent.jar"
# export HBASE_REGIONSERVER_OPTS="$HBASE_JMX_BASE
-Dcom.sun.management.jmxremote.port=10102"
# export HBASE_THRIFT_OPTS="$HBASE_JMX_BASE
-Dcom.sun.management.jmxremote.port=10103"
# export HBASE_ZOOKEEPER_OPTS="$HBASE_JMX_BASE
-Dcom.sun.management.jmxremote.port=10104"
Uncomment as appropriate.
St.Ack
On Sat, Mar 19, 2011 at 2:19 PM, Oleg Ruchovets <or...@gmail.com> wrote:
> Hi , we started our tests on cluster ( hbase 0.90.1 , hadoop append) ,
> I set HBASE_HEAPSIZE to 4000m in hbase-env.sh and got 3 processes which
> has heap size 4000m:
>
> my questions are:
> 1)What is the way to set separately heap size for these processes. In case
> I want to give to zookeper less heap size what is the properties should I
> use?
> 2)Assuming that I want to set 4000m for hbase, Does it mean that all
> these 3 processes have to have 4000m heap size? What is the advisable heap
> size for hbase?
>
> *the processes are: *
>
>
>
> hadoop 10291 1 0 Mar17 ? 00:00:18 /usr/java/default//bin/java
> -Xmx4000m -XX:+HeapDumpOnOutOfMemoryError -XX:+UseConcMarkSweepGC
> -XX:+CMSIncrementalMode -verbose:gc -XX:+PrintGCDetails
> -XX:+PrintGCTimeStamps
> -Xloggc:/opt/hbase/bin/../logs/gc/gc-hbase_10291_20110317_191315.log
> -Dhbase.log.dir=/opt/hbase/bin/../logs
> -Dhbase.log.file=hbase-hadoop-zookeeper-hadoop01.infolinks.local.log
> -Dhbase.home.dir=/opt/hbase/bin/.. -Dhbase.id.str=hadoop
> -Dhbase.root.logger=INFO,DRFA -classpath
> /opt/hbase/bin/../conf:/usr/java/default//lib/tools.jar:/opt/hbase/bin/..:/opt/hbase/bin/../hbase-0.90.1.jar:/opt/hbase/bin/../hbase-0.90.1-tests.jar:/opt/hbase/bin/../lib/avro-1.3.3.jar:/opt/hbase/bin/../lib/commons-cli-1.2.jar:/opt/hbase/bin/../lib/commons-codec-1.4.jar:/opt/hbase/bin/../lib/commons-el-1.0.jar:/opt/hbase/bin/../lib/commons-httpclient-3.1.jar:/opt/hbase/bin/../lib/commons-lang-2.5.jar:/opt/hbase/bin/../lib/commons-logging-1.1.1.jar:/opt/hbase/bin/../lib/guava-r06.jar:/opt/hbase/bin/../lib/hadoop-core-0.20.3-SNAPSHOT.jar:/opt/hbase/bin/../lib/jasper-compiler-5.5.23.jar:/opt/hbase/bin/../lib/jasper-runtime-5.5.23.jar:/opt/hbase/bin/../lib/jaxb-api-2.1.jar:/opt/hbase/bin/../lib/jersey-core-1.4.jar:/opt/hbase/bin/../lib/jersey-json-1.4.jar:/opt/hbase/bin/../lib/jersey-server-1.4.jar:/opt/hbase/bin/../lib/jetty-6.1.26.jar:/opt/hbase/bin/../lib/jetty-util-6.1.26.jar:/opt/hbase/bin/../lib/jruby-complete-1.0.3.jar:/opt/hbase/bin/../lib/jsp-2.1-6.1.14.jar:/opt/hbase/bin/../lib/jsp-api-2.1-6.1.14.jar:/opt/hbase/bin/../lib/jsr311-api-1.1.1.jar:/opt/hbase/bin/../lib/log4j-1.2.16.jar:/opt/hbase/bin/../lib/protobuf-java-2.3.0.jar:/opt/hbase/bin/../lib/servlet-api-2.5-6.1.14.jar:/opt/hbase/bin/../lib/slf4j-api-1.5.8.jar:/opt/hbase/bin/../lib/slf4j-log4j12-1.5.8.jar:/opt/hbase/bin/../lib/stax-api-1.0.1.jar:/opt/hbase/bin/../lib/thrift-0.2.0.jar:/opt/hbase/bin/../lib/zookeeper-3.3.2.jar
> org.apache.hadoop.hbase.zookeeper.HQuorumPeer start
> hadoop 10368 1 0 Mar17 pts/2 00:05:50 /usr/java/default//bin/java
> -Xmx4000m -XX:+HeapDumpOnOutOfMemoryError -XX:+UseConcMarkSweepGC
> -XX:+CMSIncrementalMode -verbose:gc -XX:+PrintGCDetails
> -XX:+PrintGCTimeStamps
> -Xloggc:/opt/hbase/bin/../logs/gc/gc-hbase_10368_20110317_191316.log
> -Dhbase.log.dir=/opt/hbase/bin/../logs
> -Dhbase.log.file=hbase-hadoop-master-hadoop01.infolinks.local.log
> -Dhbase.home.dir=/opt/hbase/bin/.. -Dhbase.id.str=hadoop
> -Dhbase.root.logger=INFO,DRFA -classpath
> /opt/hbase/bin/../conf:/usr/java/default//lib/tools.jar:/opt/hbase/bin/..:/opt/hbase/bin/../hbase-0.90.1.jar:/opt/hbase/bin/../hbase-0.90.1-tests.jar:/opt/hbase/bin/../lib/avro-1.3.3.jar:/opt/hbase/bin/../lib/commons-cli-1.2.jar:/opt/hbase/bin/../lib/commons-codec-1.4.jar:/opt/hbase/bin/../lib/commons-el-1.0.jar:/opt/hbase/bin/../lib/commons-httpclient-3.1.jar:/opt/hbase/bin/../lib/commons-lang-2.5.jar:/opt/hbase/bin/../lib/commons-logging-1.1.1.jar:/opt/hbase/bin/../lib/guava-r06.jar:/opt/hbase/bin/../lib/hadoop-core-0.20.3-SNAPSHOT.jar:/opt/hbase/bin/../lib/jasper-compiler-5.5.23.jar:/opt/hbase/bin/../lib/jasper-runtime-5.5.23.jar:/opt/hbase/bin/../lib/jaxb-api-2.1.jar:/opt/hbase/bin/../lib/jersey-core-1.4.jar:/opt/hbase/bin/../lib/jersey-json-1.4.jar:/opt/hbase/bin/../lib/jersey-server-1.4.jar:/opt/hbase/bin/../lib/jetty-6.1.26.jar:/opt/hbase/bin/../lib/jetty-util-6.1.26.jar:/opt/hbase/bin/../lib/jruby-complete-1.0.3.jar:/opt/hbase/bin/../lib/jsp-2.1-6.1.14.jar:/opt/hbase/bin/../lib/jsp-api-2.1-6.1.14.jar:/opt/hbase/bin/../lib/jsr311-api-1.1.1.jar:/opt/hbase/bin/../lib/log4j-1.2.16.jar:/opt/hbase/bin/../lib/protobuf-java-2.3.0.jar:/opt/hbase/bin/../lib/servlet-api-2.5-6.1.14.jar:/opt/hbase/bin/../lib/slf4j-api-1.5.8.jar:/opt/hbase/bin/../lib/slf4j-log4j12-1.5.8.jar:/opt/hbase/bin/../lib/stax-api-1.0.1.jar:/opt/hbase/bin/../lib/thrift-0.2.0.jar:/opt/hbase/bin/../lib/zookeeper-3.3.2.jar
> org.apache.hadoop.hbase.master.HMaster start
> hadoop 10548 1 5 Mar17 ? 02:39:01 /usr/java/default//bin/java
> -Xmx4000m -XX:+HeapDumpOnOutOfMemoryError -XX:+UseConcMarkSweepGC
> -XX:+CMSIncrementalMode -verbose:gc -XX:+PrintGCDetails
> -XX:+PrintGCTimeStamps
> -Xloggc:/opt/hbase/bin/../logs/gc/gc-hbase_10548_20110317_191318.log
> -Dhbase.log.dir=/opt/hbase/bin/../logs
> -Dhbase.log.file=hbase-hadoop-regionserver-hadoop01.infolinks.local.log
> -Dhbase.home.dir=/opt/hbase/bin/.. -Dhbase.id.str=hadoop
> -Dhbase.root.logger=INFO,DRFA -classpath
> /opt/hbase/bin/../conf:/usr/java/default//lib/tools.jar:/opt/hbase/bin/..:/opt/hbase/bin/../hbase-0.90.1.jar:/opt/hbase/bin/../hbase-0.90.1-tests.jar:/opt/hbase/bin/../lib/avro-1.3.3.jar:/opt/hbase/bin/../lib/commons-cli-1.2.jar:/opt/hbase/bin/../lib/commons-codec-1.4.jar:/opt/hbase/bin/../lib/commons-el-1.0.jar:/opt/hbase/bin/../lib/commons-httpclient-3.1.jar:/opt/hbase/bin/../lib/commons-lang-2.5.jar:/opt/hbase/bin/../lib/commons-logging-1.1.1.jar:/opt/hbase/bin/../lib/guava-r06.jar:/opt/hbase/bin/../lib/hadoop-core-0.20.3-SNAPSHOT.jar:/opt/hbase/bin/../lib/jasper-compiler-5.5.23.jar:/opt/hbase/bin/../lib/jasper-runtime-5.5.23.jar:/opt/hbase/bin/../lib/jaxb-api-2.1.jar:/opt/hbase/bin/../lib/jersey-core-1.4.jar:/opt/hbase/bin/../lib/jersey-json-1.4.jar:/opt/hbase/bin/../lib/jersey-server-1.4.jar:/opt/hbase/bin/../lib/jetty-6.1.26.jar:/opt/hbase/bin/../lib/jetty-util-6.1.26.jar:/opt/hbase/bin/../lib/jruby-complete-1.0.3.jar:/opt/hbase/bin/../lib/jsp-2.1-6.1.14.jar:/opt/hbase/bin/../lib/jsp-api-2.1-6.1.14.jar:/opt/hbase/bin/../lib/jsr311-api-1.1.1.jar:/opt/hbase/bin/../lib/log4j-1.2.16.jar:/opt/hbase/bin/../lib/protobuf-java-2.3.0.jar:/opt/hbase/bin/../lib/servlet-api-2.5-6.1.14.jar:/opt/hbase/bin/../lib/slf4j-api-1.5.8.jar:/opt/hbase/bin/../lib/slf4j-log4j12-1.5.8.jar:/opt/hbase/bin/../lib/stax-api-1.0.1.jar:/opt/hbase/bin/../lib/thrift-0.2.0.jar:/opt/hbase/bin/../lib/zookeeper-3.3.2.jar
> org.apache.hadoop.hbase.regionserver.HRegionServer start
>
> Thanks in advance
> Oleg.
>