You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sw...@apache.org on 2015/08/04 02:38:13 UTC
ambari git commit: AMBARI-12626. Ambari does not allow hard limit for
hdfs daemons to be modified. (swagle)
Repository: ambari
Updated Branches:
refs/heads/branch-2.1 bc7b80320 -> 62701f63d
AMBARI-12626. Ambari does not allow hard limit for hdfs daemons to be modified. (swagle)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/62701f63
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/62701f63
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/62701f63
Branch: refs/heads/branch-2.1
Commit: 62701f63d16d77f721bc46c97fc37b817be4e545
Parents: bc7b803
Author: Siddharth Wagle <sw...@hortonworks.com>
Authored: Mon Aug 3 17:30:18 2015 -0700
Committer: Siddharth Wagle <sw...@hortonworks.com>
Committed: Mon Aug 3 17:30:18 2015 -0700
----------------------------------------------------------------------
.../HDFS/2.1.0.2.0/configuration/hadoop-env.xml | 12 ++++++++++++
.../HDFS/2.1.0.2.0/package/scripts/params_linux.py | 3 +++
.../HDFS/2.1.0.2.0/package/templates/hdfs.conf.j2 | 4 ++--
.../HIVE/0.12.0.2.0/configuration/hive-env.xml | 14 +++++++++++++-
.../HIVE/0.12.0.2.0/package/scripts/params_linux.py | 3 +++
.../HIVE/0.12.0.2.0/package/templates/hive.conf.j2 | 4 ++--
.../2.1.0.2.0/configuration-mapred/mapred-env.xml | 12 ++++++++++++
.../YARN/2.1.0.2.0/configuration/yarn-env.xml | 13 +++++++++++++
.../YARN/2.1.0.2.0/package/scripts/params_linux.py | 6 ++++++
.../2.1.0.2.0/package/templates/mapreduce.conf.j2 | 4 ++--
.../YARN/2.1.0.2.0/package/templates/yarn.conf.j2 | 4 ++--
11 files changed, 70 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/62701f63/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hadoop-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hadoop-env.xml b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hadoop-env.xml
index d33cfda..ab8f4ce 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hadoop-env.xml
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hadoop-env.xml
@@ -155,6 +155,18 @@
<description>File path that contains the last known mount point for each data dir. This file is used to avoid creating a DFS data dir on the root drive (and filling it up) if a path was previously mounted on a drive.</description>
</property>
+ <property>
+ <name>hdfs_user_nofile_limit</name>
+ <value>128000</value>
+ <description>Max open files limit setting for HDFS user.</description>
+ </property>
+
+ <property>
+ <name>hdfs_user_nproc_limit</name>
+ <value>65536</value>
+ <description>Max number of processes limit setting for HDFS user.</description>
+ </property>
+
<!-- hadoop-env.sh -->
<property>
<name>content</name>
http://git-wip-us.apache.org/repos/asf/ambari/blob/62701f63/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
index c53508b..e978330 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
@@ -101,6 +101,9 @@ if Script.is_hdp_stack_greater_or_equal("2.2"):
ambari_libs_dir = "/var/lib/ambari-agent/lib"
limits_conf_dir = "/etc/security/limits.d"
+hdfs_user_nofile_limit = default("/configurations/hadoop-env/hdfs_user_nofile_limit", "128000")
+hdfs_user_nproc_limit = default("/configurations/hadoop-env/hdfs_user_nproc_limit", "65536")
+
create_lib_snappy_symlinks = not Script.is_hdp_stack_greater_or_equal("2.2")
if Script.is_hdp_stack_greater_or_equal("2.0") and Script.is_hdp_stack_less_than("2.1") and not OSCheck.is_suse_family():
http://git-wip-us.apache.org/repos/asf/ambari/blob/62701f63/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/templates/hdfs.conf.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/templates/hdfs.conf.j2 b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/templates/hdfs.conf.j2
index d58a6f5..fad5621 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/templates/hdfs.conf.j2
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/templates/hdfs.conf.j2
@@ -31,5 +31,5 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-{{hdfs_user}} - nofile 32768
-{{hdfs_user}} - nproc 65536
+{{hdfs_user}} - nofile {{hdfs_user_nofile_limit}}
+{{hdfs_user}} - nproc {{hdfs_user_nproc_limit}}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/62701f63/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml
index 64653b5..d201df5 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml
@@ -96,7 +96,19 @@
<property-type>USER</property-type>
<description>WebHCat User.</description>
</property>
-
+
+ <property>
+ <name>hive_user_nofile_limit</name>
+ <value>32000</value>
+ <description>Max open files limit setting for HIVE user.</description>
+ </property>
+
+ <property>
+ <name>hive_user_nproc_limit</name>
+ <value>16000</value>
+ <description>Max number of processes limit setting for HIVE user.</description>
+ </property>
+
<!-- hive-env.sh -->
<property>
<name>content</name>
http://git-wip-us.apache.org/repos/asf/ambari/blob/62701f63/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
index 35b896f..daca7c5 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
@@ -87,6 +87,9 @@ config_dir = '/etc/hcatalog/conf'
hcat_lib = '/usr/lib/hcatalog/share/hcatalog'
webhcat_bin_dir = '/usr/lib/hcatalog/sbin'
+hive_user_nofile_limit = default("/configurations/hive-env/hive_user_nofile_limit", "32000")
+hive_user_nproc_limit = default("/configurations/hive-env/hive_user_nproc_limit", "16000")
+
# use the directories from status_params as they are already calculated for
# the correct version of HDP
hadoop_conf_dir = status_params.hadoop_conf_dir
http://git-wip-us.apache.org/repos/asf/ambari/blob/62701f63/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hive.conf.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hive.conf.j2 b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hive.conf.j2
index 8b7f8bd..5af53d0 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hive.conf.j2
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hive.conf.j2
@@ -31,5 +31,5 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-{{hive_user}} - nofile 32000
-{{hive_user}} - nproc 16000
+{{hive_user}} - nofile {{hive_user_nofile_limit}}
+{{hive_user}} - nproc {{hive_user_nproc_limit}}
http://git-wip-us.apache.org/repos/asf/ambari/blob/62701f63/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-env.xml b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-env.xml
index 14ae20b..b387102 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-env.xml
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-env.xml
@@ -42,6 +42,18 @@
<value>900</value>
<description>Value for JobHistoryServer heap_size variable in hadoop-env.sh</description>
</property>
+
+ <property>
+ <name>mapred_user_nofile_limit</name>
+ <value>32768</value>
+ <description>Max open files limit setting for MAPREDUCE user.</description>
+ </property>
+
+ <property>
+ <name>mapred_user_nproc_limit</name>
+ <value>65536</value>
+ <description>Max number of processes limit setting for MAPREDUCE user.</description>
+ </property>
<!-- mapred-env.sh -->
<property>
http://git-wip-us.apache.org/repos/asf/ambari/blob/62701f63/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-env.xml b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-env.xml
index c16c3ac..a1dfa57 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-env.xml
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-env.xml
@@ -62,6 +62,19 @@
<value>false</value>
<description>Set to false by default, needs to be set to true in stacks that use Ranger Yarn Plugin</description>
</property>
+
+ <property>
+ <name>yarn_user_nofile_limit</name>
+ <value>32768</value>
+ <description>Max open files limit setting for YARN user.</description>
+ </property>
+
+ <property>
+ <name>yarn_user_nproc_limit</name>
+ <value>65536</value>
+ <description>Max number of processes limit setting for YARN user.</description>
+ </property>
+
<!-- yarn-env.sh -->
<property>
<name>content</name>
http://git-wip-us.apache.org/repos/asf/ambari/blob/62701f63/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
index 4443476..f3914f3 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
@@ -101,6 +101,12 @@ if Script.is_hdp_stack_greater_or_equal("2.2"):
hadoop_conf_secure_dir = os.path.join(hadoop_conf_dir, "secure")
limits_conf_dir = "/etc/security/limits.d"
+yarn_user_nofile_limit = default("/configurations/yarn-env/yarn_user_nofile_limit", "32768")
+yarn_user_nproc_limit = default("/configurations/yarn-env/yarn_user_nproc_limit", "65536")
+
+mapred_user_nofile_limit = default("/configurations/mapred-env/mapred_user_nofile_limit", "32768")
+mapred_user_nproc_limit = default("/configurations/mapred-env/mapred_user_nproc_limit", "65536")
+
execute_path = os.environ['PATH'] + os.pathsep + hadoop_bin_dir + os.pathsep + yarn_container_bin
ulimit_cmd = "ulimit -c unlimited;"
http://git-wip-us.apache.org/repos/asf/ambari/blob/62701f63/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/mapreduce.conf.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/mapreduce.conf.j2 b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/mapreduce.conf.j2
index b996645..ae8e6d5 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/mapreduce.conf.j2
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/mapreduce.conf.j2
@@ -31,5 +31,5 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-{{mapred_user}} - nofile 32768
-{{mapred_user}} - nproc 65536
+{{mapred_user}} - nofile {{mapred_user_nofile_limit}}
+{{mapred_user}} - nproc {{mapred_user_nproc_limit}}
http://git-wip-us.apache.org/repos/asf/ambari/blob/62701f63/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/yarn.conf.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/yarn.conf.j2 b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/yarn.conf.j2
index 3bd7a45..1063099 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/yarn.conf.j2
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/yarn.conf.j2
@@ -31,5 +31,5 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-{{yarn_user}} - nofile 32768
-{{yarn_user}} - nproc 65536
+{{yarn_user}} - nofile {{yarn_user_nofile_limit}}
+{{yarn_user}} - nproc {{yarn_user_nproc_limit}}