You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2014/11/25 08:57:53 UTC
hbase git commit: HBASE-12569 Update scripts to control
MaxDirectMemorySize via env vars
Repository: hbase
Updated Branches:
refs/heads/master e83082a88 -> f2be914f7
HBASE-12569 Update scripts to control MaxDirectMemorySize via env vars
Signed-off-by: stack <st...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f2be914f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f2be914f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f2be914f
Branch: refs/heads/master
Commit: f2be914f73d39d288d40147ff1e582ad5a7989f2
Parents: e83082a
Author: Patrick White <pw...@fb.com>
Authored: Mon Nov 24 17:58:03 2014 -0800
Committer: stack <st...@apache.org>
Committed: Mon Nov 24 23:57:46 2014 -0800
----------------------------------------------------------------------
bin/hbase | 34 ++++++++++++++++++++--------------
bin/hbase.cmd | 8 +++++++-
conf/hbase-env.cmd | 6 ++++++
conf/hbase-env.sh | 10 ++++++----
src/main/docbkx/book.xml | 4 ++--
5 files changed, 41 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/f2be914f/bin/hbase
----------------------------------------------------------------------
diff --git a/bin/hbase b/bin/hbase
index 4573c57..31fd232 100755
--- a/bin/hbase
+++ b/bin/hbase
@@ -114,18 +114,23 @@ if [ -f "$HBASE_HOME/conf/hbase-env-$COMMAND.sh" ]; then
. "$HBASE_HOME/conf/hbase-env-$COMMAND.sh"
fi
-# check envvars which might override default args
-if [ "$HBASE_HEAPSIZE" != "" ]; then
- SUFFIX="m"
- if [ "${HBASE_HEAPSIZE: -1}" == "m" ] || [ "${HBASE_HEAPSIZE: -1}" == "M" ]; then
- SUFFIX=""
- fi
- if [ "${HBASE_HEAPSIZE: -1}" == "g" ] || [ "${HBASE_HEAPSIZE: -1}" == "G" ]; then
- SUFFIX=""
- fi
- #echo "run with heapsize $HBASE_HEAPSIZE"
- JAVA_HEAP_MAX="-Xmx""$HBASE_HEAPSIZE""$SUFFIX"
- #echo $JAVA_HEAP_MAX
+add_size_suffix() {
+ # add an 'm' suffix if the argument is missing one, otherwise use whats there
+ local val="$1"
+ local lastchar=${val: -1}
+ if [[ "mMgG" == *$lastchar* ]]; then
+ echo $val
+ else
+ echo ${val}m
+ fi
+}
+
+if [[ -n "$HBASE_HEAPSIZE" ]]; then
+ JAVA_HEAP_MAX="-Xmx$(add_size_suffix $HBASE_HEAPSIZE)"
+fi
+
+if [[ -n "$HBASE_OFFHEAPSIZE" ]]; then
+ JAVA_OFFHEAP_MAX="-XX:MaxDirectMemorySize=$(add_size_suffix $HBASE_OFFHEAPSIZE)"
fi
# so that filenames w/ spaces are handled correctly in loops below
@@ -381,10 +386,11 @@ else
HBASE_OPTS="$HBASE_OPTS -Dhbase.security.logger=${HBASE_SECURITY_LOGGER:-INFO,NullAppender}"
fi
+HEAP_SETTINGS="$JAVA_HEAP_MAX $JAVA_OFFHEAP_MAX"
# Exec unless HBASE_NOEXEC is set.
export CLASSPATH
if [ "${HBASE_NOEXEC}" != "" ]; then
- "$JAVA" -Dproc_$COMMAND -XX:OnOutOfMemoryError="kill -9 %p" $JAVA_HEAP_MAX $HBASE_OPTS $CLASS "$@"
+ "$JAVA" -Dproc_$COMMAND -XX:OnOutOfMemoryError="kill -9 %p" $HEAP_SETTINGS $HBASE_OPTS $CLASS "$@"
else
- exec "$JAVA" -Dproc_$COMMAND -XX:OnOutOfMemoryError="kill -9 %p" $JAVA_HEAP_MAX $HBASE_OPTS $CLASS "$@"
+ exec "$JAVA" -Dproc_$COMMAND -XX:OnOutOfMemoryError="kill -9 %p" $HEAP_SETTINGS $HBASE_OPTS $CLASS "$@"
fi
http://git-wip-us.apache.org/repos/asf/hbase/blob/f2be914f/bin/hbase.cmd
----------------------------------------------------------------------
diff --git a/bin/hbase.cmd b/bin/hbase.cmd
index a173b6c..db7d856 100644
--- a/bin/hbase.cmd
+++ b/bin/hbase.cmd
@@ -88,12 +88,17 @@ if "%hbase-command%"=="" (
)
set JAVA_HEAP_MAX=-Xmx1000m
+set JAVA_OFFHEAP_MAX=""
rem check envvars which might override default args
if defined HBASE_HEAPSIZE (
set JAVA_HEAP_MAX=-Xmx%HBASE_HEAPSIZE%m
)
+if defined HBASE_OFFHEAPSIZE (
+ set JAVA_OFFHEAP_MAX=-XX:MaxDirectMemory=%HBASE_OFFHEAPSIZE%m
+)
+
set CLASSPATH=%HBASE_CONF_DIR%;%JAVA_HOME%\lib\tools.jar
rem Add maven target directory
@@ -288,7 +293,8 @@ if not defined HBASE_SECURITY_LOGGER (
)
set HBASE_OPTS=%HBASE_OPTS% -Dhbase.security.logger="%HBASE_SECURITY_LOGGER%"
-set java_arguments=%JAVA_HEAP_MAX% %HBASE_OPTS% -classpath "%CLASSPATH%" %CLASS% %hbase-command-arguments%
+set HEAP_SETTINGS="%JAVA_HEAP_MAX% %JAVA_OFFHEAP_MAX%"
+set java_arguments=%HEAP_SETTINGS% %HBASE_OPTS% -classpath "%CLASSPATH%" %CLASS% %hbase-command-arguments%
if defined service_entry (
call :makeServiceXml %java_arguments%
http://git-wip-us.apache.org/repos/asf/hbase/blob/f2be914f/conf/hbase-env.cmd
----------------------------------------------------------------------
diff --git a/conf/hbase-env.cmd b/conf/hbase-env.cmd
index 26570db..bf7f25a 100644
--- a/conf/hbase-env.cmd
+++ b/conf/hbase-env.cmd
@@ -27,6 +27,12 @@
@rem The maximum amount of heap to use, in MB. Default is 1000.
@rem set HBASE_HEAPSIZE=1000
+@rem Uncomment below if you intend to use off heap cache.
+@rem set HBASE_OFFHEAPSIZE=1000
+
+@rem For example, to allocate 8G of offheap, to 8G:
+@rem etHBASE_OFFHEAPSIZE=8G
+
@rem Extra Java runtime options.
@rem Below are what we set by default. May only work with SUN JVM.
@rem For more on why as well as other possible settings,
http://git-wip-us.apache.org/repos/asf/hbase/blob/f2be914f/conf/hbase-env.sh
----------------------------------------------------------------------
diff --git a/conf/hbase-env.sh b/conf/hbase-env.sh
index 2d10609..1693b17 100644
--- a/conf/hbase-env.sh
+++ b/conf/hbase-env.sh
@@ -34,6 +34,12 @@
# The maximum amount of heap to use, in MB. Default is 1000.
# export HBASE_HEAPSIZE=1000
+# Uncomment below if you intend to use off heap cache.
+# export HBASE_OFFHEAPSIZE=1000
+
+# For example, to allocate 8G of offheap, to 8G:
+# export HBASE_OFFHEAPSIZE=8G
+
# Extra Java runtime options.
# Below are what we set by default. May only work with SUN JVM.
# For more on why as well as other possible settings,
@@ -66,10 +72,6 @@ export HBASE_OPTS="-XX:+UseConcMarkSweepGC"
# If FILE-PATH is not replaced, the log file(.gc) would still be generated in the HBASE_LOG_DIR .
# export CLIENT_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:<FILE-PATH> -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=1 -XX:GCLogFileSize=512M"
-# Uncomment below if you intend to use off heap cache.
-# export HBASE_OPTS="$HBASE_OPTS -XX:MaxDirectMemorySize=SET_THIS_TO_HOW_MANY_GIGS_OF_OFFHEAP"
-# For example, to allocate 8G of offheap, set SET_THIS_TO_HOW_MANY_GIGS_OF_OFFHEAP to 8G as in:
-# export HBASE_OPTS="$HBASE_OPTS -XX:MaxDirectMemorySize=8G"
# See the package documentation for org.apache.hadoop.hbase.io.hfile for other configurations
# needed setting up off-heap block caching.
http://git-wip-us.apache.org/repos/asf/hbase/blob/f2be914f/src/main/docbkx/book.xml
----------------------------------------------------------------------
diff --git a/src/main/docbkx/book.xml b/src/main/docbkx/book.xml
index dd777fd..862d467 100644
--- a/src/main/docbkx/book.xml
+++ b/src/main/docbkx/book.xml
@@ -2417,12 +2417,12 @@ rs.close();
</para>
<step>
<para>First, edit the RegionServer's <filename>hbase-env.sh</filename> and set
- <varname>-XX:MaxDirectMemorySize</varname> to a value greater than the offheap size wanted, in
+ <varname>HBASE_OFFHEAPSIZE</varname> to a value greater than the offheap size wanted, in
this case, 4 GB (expressed as 4G). Lets set it to 5G. That'll be 4G
for our offheap cache and 1G for any other uses of offheap memory (there are
other users of offheap memory other than BlockCache; e.g. DFSClient
in RegionServer can make use of offheap memory). See <xref linkend="direct.memory" />.</para>
- <programlisting>-XX:MaxDirectMemorySize=5G</programlisting>
+ <programlisting>HBASE_OFFHEAPSIZE=5G</programlisting>
</step>
<step>
<para>Next, add the following configuration to the RegionServer's