You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sm...@apache.org on 2013/08/15 07:51:22 UTC
git commit: AMBARI-2907. yarn-env.sh is missing few critical
attributes. (smohanty)
Updated Branches:
refs/heads/trunk 9c3940bd0 -> 32399749c
AMBARI-2907. yarn-env.sh is missing few critical attributes. (smohanty)
Project: http://git-wip-us.apache.org/repos/asf/incubator-ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ambari/commit/32399749
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ambari/tree/32399749
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ambari/diff/32399749
Branch: refs/heads/trunk
Commit: 32399749c9394cef0088328583f192c6e548165c
Parents: 9c3940b
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Wed Aug 14 22:50:58 2013 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Wed Aug 14 22:50:58 2013 -0700
----------------------------------------------------------------------
.../puppet/modules/hdp-yarn/manifests/params.pp | 9 +-
.../modules/hdp-yarn/templates/yarn-env.sh.erb | 172 +++++++++++--------
.../services/YARN/configuration/global.xml | 15 ++
.../services/YARN/configuration/global.xml | 15 ++
4 files changed, 130 insertions(+), 81 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/32399749/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/params.pp
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/params.pp b/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/params.pp
index d09483b..8916196 100644
--- a/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/params.pp
+++ b/ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/params.pp
@@ -32,12 +32,11 @@ class hdp-yarn::params(
## yarn-env
$hadoop_libexec_dir = hdp_default("yarn/yarn-env/hadoop_libexec_dir","/usr/lib/hadoop/libexec")
-
- $hadoop_common_home = hdp_default("yarn/yarn-env/hadoop_common_home","/usr/lib/hadoop")
- $hadoop_hdfs_home = hdp_default("yarn/yarn-env/hadoop_hdfs_home","/usr/lib/hadoop-hdfs")
- $hadoop_mapred_home = hdp_default("yarn/yarn-env/hadoop_mapred_home","/usr/lib/hadoop-mapreduce")
$hadoop_yarn_home = hdp_default("yarn/yarn-env/hadoop_yarn_home","/usr/lib/hadoop-yarn")
-
+ $yarn_heapsize = hdp_default("yarn/yarn-env/yarn_heapsize","1024")
+ $resourcemanager_heapsize = hdp_default("yarn/yarn-env/resourcemanager_heapsize","1024")
+ $nodemanager_heapsize = hdp_default("yarn/yarn-env/nodemanager_heapsize","1024")
+
$yarn_log_dir_prefix = hdp_default("hadoop/yarn-env/yarn_log_dir_prefix","/var/log/hadoop-yarn")
$yarn_pid_dir_prefix = hdp_default("hadoop/yarn-env/yarn_pid_dir_prefix","/var/run/hadoop-yarn")
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/32399749/ambari-agent/src/main/puppet/modules/hdp-yarn/templates/yarn-env.sh.erb
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/puppet/modules/hdp-yarn/templates/yarn-env.sh.erb b/ambari-agent/src/main/puppet/modules/hdp-yarn/templates/yarn-env.sh.erb
index d1bd482..4bf1779 100644
--- a/ambari-agent/src/main/puppet/modules/hdp-yarn/templates/yarn-env.sh.erb
+++ b/ambari-agent/src/main/puppet/modules/hdp-yarn/templates/yarn-env.sh.erb
@@ -16,84 +16,104 @@
# * limitations under the License.
# */
-# Set Hadoop-specific environment variables here.
-
-# The only required environment variable is JAVA_HOME. All others are
-# optional. When running a distributed configuration it is best to
-# set JAVA_HOME in this file, so that it is correctly defined on
-# remote nodes.
-
-export HADOOP_COMMON_HOME=<%=scope.function_hdp_template_var("hadoop_common_home")%>
-export HADOOP_HDFS_HOME=<%=scope.function_hdp_template_var("hadoop_hdfs_home")%>
-export HADOOP_MAPRED_HOME=<%=scope.function_hdp_template_var("hadoop_mapred_home")%>
export HADOOP_YARN_HOME=<%=scope.function_hdp_template_var("hadoop_yarn_home")%>
-
export YARN_LOG_DIR=<%=scope.function_hdp_template_var("yarn_log_dir_prefix")%>/$USER
export YARN_PID_DIR=<%=scope.function_hdp_template_var("yarn_pid_dir_prefix")%>/$USER
-
-export HADOOP_LIBEXEC_DIR=/usr/lib/hadoop/libexec
-
-# The java implementation to use. Required.
+export HADOOP_LIBEXEC_DIR=<%=scope.function_hdp_template_var("hadoop_libexec_dir")%>
export JAVA_HOME=<%=scope.function_hdp_java_home()%>
-export HADOOP_HOME_WARN_SUPPRESS=1
-
-# Hadoop Configuration Directory
-#TODO: if env var set that can cause problems
-export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/etc/hadoop/conf}
-
-# The maximum amount of heap to use, in MB. Default is 1000.
-export HADOOP_HEAPSIZE="1024"
-
-export HADOOP_NAMENODE_INIT_HEAPSIZE="-Xms1024m"
-
-# Extra Java runtime options. Empty by default.
-export HADOOP_OPTS="-Djava.net.preferIPv4Stack=true ${HADOOP_OPTS}"
-
-# Command specific options appended to HADOOP_OPTS when specified
-export HADOOP_NAMENODE_OPTS="-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile=/var/log/hadoop/$USER/hs_err_pid%p.log -XX:NewSize=200m -XX:MaxNewSize=640m -Xloggc:/var/log/hadoop/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xms1024m -Xmx1024m -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_NAMENODE_OPTS}"
-HADOOP_JOBTRACKER_OPTS="-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile=/var/log/hadoop/$USER/hs_err_pid%p.log -XX:NewSize=200m -XX:MaxNewSize=200m -Xloggc:/var/log/hadoop/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xmx1024m -Dhadoop.security.logger=INFO,DRFAS -Dmapred.audit.logger=INFO,MRAUDIT -Dhadoop.mapreduce.jobsummary.logger=INFO,JSA ${HADOOP_JOBTRACKER_OPTS}"
-
-HADOOP_TASKTRACKER_OPTS="-server -Xmx1024m -Dhadoop.security.logger=ERROR,console -Dmapred.audit.logger=ERROR,console ${HADOOP_TASKTRACKER_OPTS}"
-HADOOP_DATANODE_OPTS="-Xmx1024m -Dhadoop.security.logger=ERROR,DRFAS ${HADOOP_DATANODE_OPTS}"
-HADOOP_BALANCER_OPTS="-server -Xmx1024m ${HADOOP_BALANCER_OPTS}"
-
-export HADOOP_SECONDARYNAMENODE_OPTS="-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile=/var/log/hadoop/$USER/hs_err_pid%p.log -XX:NewSize=200m -XX:MaxNewSize=640m -Xloggc:/var/log/hadoop/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps ${HADOOP_NAMENODE_INIT_HEAPSIZE} -Xmx1024m -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_SECONDARYNAMENODE_OPTS}"
-
-# The following applies to multiple commands (fs, dfs, fsck, distcp etc)
-export HADOOP_CLIENT_OPTS="-Xmx128m ${HADOOP_CLIENT_OPTS}"
-#HADOOP_JAVA_PLATFORM_OPTS="-XX:-UsePerfData ${HADOOP_JAVA_PLATFORM_OPTS}"
-
-# On secure datanodes, user to run the datanode as after dropping privileges
-export HADOOP_SECURE_DN_USER=hdfs
-
-# Extra ssh options. Empty by default.
-export HADOOP_SSH_OPTS="-o ConnectTimeout=5 -o SendEnv=HADOOP_CONF_DIR"
-
-# Where log files are stored. $HADOOP_HOME/logs by default.
-export HADOOP_LOG_DIR=/var/log/hadoop/$USER
-
-
-# Where log files are stored in the secure data environment.
-export HADOOP_SECURE_DN_LOG_DIR=/var/log/hadoop/$HADOOP_SECURE_DN_USER
-
-# File naming remote slave hosts. $HADOOP_HOME/conf/slaves by default.
-# export HADOOP_SLAVES=${HADOOP_HOME}/conf/slaves
-
-# host:path where hadoop code should be rsync'd from. Unset by default.
-# export HADOOP_MASTER=master:/home/$USER/src/hadoop
-
-# Seconds to sleep between slave commands. Unset by default. This
-# can be useful in large clusters, where, e.g., slave rsyncs can
-# otherwise arrive faster than the master can service them.
-# export HADOOP_SLAVE_SLEEP=0.1
-
-# The directory where pid files are stored. /tmp by default.
-export HADOOP_PID_DIR=/var/run/hadoop/$USER
-export HADOOP_SECURE_DN_PID_DIR=/var/run/hadoop/$HADOOP_SECURE_DN_USER
-
-# A string representing this instance of hadoop. $USER by default.
-export HADOOP_IDENT_STRING=$USER
-
-# The scheduling priority for daemon processes. See 'man nice'.
-# export HADOOP_NICENESS=10
+# User for YARN daemons
+export HADOOP_YARN_USER=${HADOOP_YARN_USER:-yarn}
+
+# resolve links - $0 may be a softlink
+export YARN_CONF_DIR="${YARN_CONF_DIR:-$HADOOP_YARN_HOME/conf}"
+
+# some Java parameters
+# export JAVA_HOME=/home/y/libexec/jdk1.6.0/
+if [ "$JAVA_HOME" != "" ]; then
+ #echo "run java in $JAVA_HOME"
+ JAVA_HOME=$JAVA_HOME
+fi
+
+if [ "$JAVA_HOME" = "" ]; then
+ echo "Error: JAVA_HOME is not set."
+ exit 1
+fi
+
+JAVA=$JAVA_HOME/bin/java
+JAVA_HEAP_MAX=-Xmx1000m
+
+# For setting YARN specific HEAP sizes please use this
+# Parameter and set appropriately
+YARN_HEAPSIZE=<%=scope.function_hdp_template_var("yarn_heapsize")%>
+
+# check envvars which might override default args
+if [ "$YARN_HEAPSIZE" != "" ]; then
+ JAVA_HEAP_MAX="-Xmx""$YARN_HEAPSIZE""m"
+fi
+
+# Resource Manager specific parameters
+
+# Specify the max Heapsize for the ResourceManager using a numerical value
+# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set
+# the value to 1000.
+# This value will be overridden by an Xmx setting specified in either YARN_OPTS
+# and/or YARN_RESOURCEMANAGER_OPTS.
+# If not specified, the default value will be picked from either YARN_HEAPMAX
+# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.
+export YARN_RESOURCEMANAGER_HEAPSIZE=<%=scope.function_hdp_template_var("resourcemanager_heapsize")%>
+
+# Specify the JVM options to be used when starting the ResourceManager.
+# These options will be appended to the options specified as YARN_OPTS
+# and therefore may override any similar flags set in YARN_OPTS
+#export YARN_RESOURCEMANAGER_OPTS=
+
+# Node Manager specific parameters
+
+# Specify the max Heapsize for the NodeManager using a numerical value
+# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set
+# the value to 1000.
+# This value will be overridden by an Xmx setting specified in either YARN_OPTS
+# and/or YARN_NODEMANAGER_OPTS.
+# If not specified, the default value will be picked from either YARN_HEAPMAX
+# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.
+export YARN_NODEMANAGER_HEAPSIZE=<%=scope.function_hdp_template_var("nodemanager_heapsize")%>
+
+# Specify the JVM options to be used when starting the NodeManager.
+# These options will be appended to the options specified as YARN_OPTS
+# and therefore may override any similar flags set in YARN_OPTS
+#export YARN_NODEMANAGER_OPTS=
+
+# so that filenames w/ spaces are handled correctly in loops below
+IFS=
+
+
+# default log directory & file
+if [ "$YARN_LOG_DIR" = "" ]; then
+ YARN_LOG_DIR="$HADOOP_YARN_HOME/logs"
+fi
+if [ "$YARN_LOGFILE" = "" ]; then
+ YARN_LOGFILE='yarn.log'
+fi
+
+# default policy file for service-level authorization
+if [ "$YARN_POLICYFILE" = "" ]; then
+ YARN_POLICYFILE="hadoop-policy.xml"
+fi
+
+# restore ordinary behaviour
+unset IFS
+
+
+YARN_OPTS="$YARN_OPTS -Dhadoop.log.dir=$YARN_LOG_DIR"
+YARN_OPTS="$YARN_OPTS -Dyarn.log.dir=$YARN_LOG_DIR"
+YARN_OPTS="$YARN_OPTS -Dhadoop.log.file=$YARN_LOGFILE"
+YARN_OPTS="$YARN_OPTS -Dyarn.log.file=$YARN_LOGFILE"
+YARN_OPTS="$YARN_OPTS -Dyarn.home.dir=$YARN_COMMON_HOME"
+YARN_OPTS="$YARN_OPTS -Dyarn.id.str=$YARN_IDENT_STRING"
+YARN_OPTS="$YARN_OPTS -Dhadoop.root.logger=${YARN_ROOT_LOGGER:-INFO,console}"
+YARN_OPTS="$YARN_OPTS -Dyarn.root.logger=${YARN_ROOT_LOGGER:-INFO,console}"
+if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then
+ YARN_OPTS="$YARN_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH"
+fi
+YARN_OPTS="$YARN_OPTS -Dyarn.policy.file=$YARN_POLICYFILE"
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/32399749/ambari-server/src/main/resources/stacks/HDP/2.0.5/services/YARN/configuration/global.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.5/services/YARN/configuration/global.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.5/services/YARN/configuration/global.xml
index 7ddd6e3..edd1636 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.5/services/YARN/configuration/global.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.5/services/YARN/configuration/global.xml
@@ -46,4 +46,19 @@
<value>yarn</value>
<description>YARN User</description>
</property>
+ <property>
+ <name>yarn_heapsize</name>
+ <value>1024</value>
+ <description>Max heapsize for all YARN components using a numerical value in the scale of MB</description>
+ </property>
+ <property>
+ <name>resourcemanager_heapsize</name>
+ <value>1024</value>
+ <description>Max heapsize for ResourceManager using a numerical value in the scale of MB</description>
+ </property>
+ <property>
+ <name>nodemanager_heapsize</name>
+ <value>1024</value>
+ <description>Max heapsize for NodeManager using a numerical value in the scale of MB</description>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/32399749/ambari-server/src/main/resources/stacks/HDPLocal/2.0.5/services/YARN/configuration/global.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPLocal/2.0.5/services/YARN/configuration/global.xml b/ambari-server/src/main/resources/stacks/HDPLocal/2.0.5/services/YARN/configuration/global.xml
index 7ddd6e3..7d43a97 100644
--- a/ambari-server/src/main/resources/stacks/HDPLocal/2.0.5/services/YARN/configuration/global.xml
+++ b/ambari-server/src/main/resources/stacks/HDPLocal/2.0.5/services/YARN/configuration/global.xml
@@ -46,4 +46,19 @@
<value>yarn</value>
<description>YARN User</description>
</property>
+ <property>
+ <name>yarn_heapsize</name>
+ <value>1000</value>
+ <description>Max heapsize for all YARN components using a numerical value in the scale of MB</description>
+ </property>
+ <property>
+ <name>resourcemanager_heapsize</name>
+ <value>1000</value>
+ <description>Max heapsize for ResourceManager using a numerical value in the scale of MB</description>
+ </property>
+ <property>
+ <name>nodemanager_heapsize</name>
+ <value>1000</value>
+ <description>Max heapsize for NodeManager using a numerical value in the scale of MB</description>
+ </property>
</configuration>