You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by nc...@apache.org on 2015/09/30 16:46:10 UTC
[50/50] [abbrv] ambari git commit: AMBARI-12813 Increase OS kernel
parameters for hbase (Juanjo Marron via dsen)
AMBARI-12813 Increase OS kernel parameters for hbase (Juanjo Marron via dsen)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/bff61b83
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/bff61b83
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/bff61b83
Branch: refs/heads/branch-dev-patch-upgrade
Commit: bff61b83e962dfc7a8c7c9f8198bd038f16b6ec7
Parents: b3d36fb
Author: Dmytro Sen <ds...@apache.org>
Authored: Wed Sep 30 17:41:48 2015 +0300
Committer: Dmytro Sen <ds...@apache.org>
Committed: Wed Sep 30 17:41:48 2015 +0300
----------------------------------------------------------------------
.../0.96.0.2.0/configuration/hbase-env.xml | 12 ++++++-
.../HBASE/0.96.0.2.0/package/scripts/hbase.py | 20 +++++++++--
.../0.96.0.2.0/package/scripts/params_linux.py | 4 +++
.../0.96.0.2.0/package/scripts/status_params.py | 1 +
.../0.96.0.2.0/package/templates/hbase.conf.j2 | 35 ++++++++++++++++++++
.../stacks/2.0.6/HBASE/test_hbase_client.py | 28 ++++++++++++++--
.../stacks/2.0.6/HBASE/test_hbase_master.py | 34 ++++++++++++++++++-
.../2.0.6/HBASE/test_hbase_regionserver.py | 33 ++++++++++++++++++
.../2.0.6/HBASE/test_phoenix_queryserver.py | 24 +++++++++++++-
9 files changed, 182 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/bff61b83/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/configuration/hbase-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/configuration/hbase-env.xml b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/configuration/hbase-env.xml
index 242e9a7..f1dd64f 100644
--- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/configuration/hbase-env.xml
+++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/configuration/hbase-env.xml
@@ -86,12 +86,22 @@ and the -Xmn ratio (hbase_regionserver_xmn_ratio) exceeds this value.
<increment-step>256</increment-step>
</value-attributes>
</property>
- <property>
+ <property>
<name>hbase_user</name>
<value>hbase</value>
<property-type>USER</property-type>
<description>HBase User Name.</description>
</property>
+ <property>
+ <name>hbase_user_nofile_limit</name>
+ <value>32000</value>
+ <description>Max open files limit setting for HBASE user.</description>
+ </property>
+ <property>
+ <name>hbase_user_nproc_limit</name>
+ <value>16000</value>
+ <description>Max number of processes limit setting for HBASE user.</description>
+ </property>
<!-- hbase-env.sh -->
<property>
http://git-wip-us.apache.org/repos/asf/ambari/blob/bff61b83/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase.py
index 991f4e7..6b5369e 100644
--- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase.py
+++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase.py
@@ -55,7 +55,7 @@ def hbase(name=None):
group = params.user_group,
recursive = True
)
-
+
parent_dir = os.path.dirname(params.tmp_dir)
# In case if we have several placeholders in path
while ("${" in parent_dir):
@@ -119,8 +119,22 @@ def hbase(name=None):
owner = params.hbase_user,
content=InlineTemplate(params.hbase_env_sh_template),
group = params.user_group,
- )
-
+ )
+
+ # On some OS this folder could be not exists, so we will create it before pushing there files
+ Directory(params.limits_conf_dir,
+ recursive=True,
+ owner='root',
+ group='root'
+ )
+
+ File(os.path.join(params.limits_conf_dir, 'hbase.conf'),
+ owner='root',
+ group='root',
+ mode=0644,
+ content=Template("hbase.conf.j2")
+ )
+
hbase_TemplateConfig( params.metric_prop_file_name,
tag = 'GANGLIA-MASTER' if name == 'master' else 'GANGLIA-RS'
)
http://git-wip-us.apache.org/repos/asf/ambari/blob/bff61b83/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py
index f22b035..635be5e 100644
--- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py
@@ -75,6 +75,10 @@ if Script.is_hdp_stack_greater_or_equal("2.2"):
hbase_conf_dir = status_params.hbase_conf_dir
+limits_conf_dir = status_params.limits_conf_dir
+
+hbase_user_nofile_limit = default("/configurations/hbase-env/hbase_user_nofile_limit", "32000")
+hbase_user_nproc_limit = default("/configurations/hbase-env/hbase_user_nproc_limit", "16000")
# no symlink for phoenix-server at this point
phx_daemon_script = '/usr/hdp/current/phoenix-server/bin/queryserver.py'
http://git-wip-us.apache.org/repos/asf/ambari/blob/bff61b83/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/status_params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/status_params.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/status_params.py
index 084ee06..014e8d7 100644
--- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/status_params.py
+++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/status_params.py
@@ -50,5 +50,6 @@ else:
tmp_dir = Script.get_tmp_dir()
hbase_conf_dir = "/etc/hbase/conf"
+ limits_conf_dir = "/etc/security/limits.d"
if Script.is_hdp_stack_greater_or_equal("2.2"):
hbase_conf_dir = format("/usr/hdp/current/{component_directory}/conf")
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/bff61b83/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hbase.conf.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hbase.conf.j2 b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hbase.conf.j2
new file mode 100644
index 0000000..3580db0
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hbase.conf.j2
@@ -0,0 +1,35 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{{hbase_user}} - nofile {{hbase_user_nofile_limit}}
+{{hbase_user}} - nproc {{hbase_user_nproc_limit}}
http://git-wip-us.apache.org/repos/asf/ambari/blob/bff61b83/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_client.py b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_client.py
index 9959874..8d96849 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_client.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_client.py
@@ -90,6 +90,17 @@ class TestHBaseClient(RMFTestCase):
owner = 'hbase',
group = 'hadoop'
)
+ self.assertResourceCalled('Directory', '/etc/security/limits.d',
+ owner = 'root',
+ group = 'root',
+ recursive = True,
+ )
+ self.assertResourceCalled('File', '/etc/security/limits.d/hbase.conf',
+ content = Template('hbase.conf.j2'),
+ owner = 'root',
+ group = 'root',
+ mode = 0644,
+ )
self.assertResourceCalled('TemplateConfig', '/etc/hbase/conf/hadoop-metrics2-hbase.properties',
owner = 'hbase',
template_tag = 'GANGLIA-RS',
@@ -167,9 +178,20 @@ class TestHBaseClient(RMFTestCase):
group = 'hadoop',
)
self.assertResourceCalled('File', '/etc/hbase/conf/hbase-env.sh',
- content = InlineTemplate(self.getConfig()['configurations']['hbase-env']['content']),
- owner = 'hbase',
- group = 'hadoop',
+ content = InlineTemplate(self.getConfig()['configurations']['hbase-env']['content']),
+ owner = 'hbase',
+ group = 'hadoop',
+ )
+ self.assertResourceCalled('Directory', '/etc/security/limits.d',
+ owner = 'root',
+ group = 'root',
+ recursive = True,
+ )
+ self.assertResourceCalled('File', '/etc/security/limits.d/hbase.conf',
+ content = Template('hbase.conf.j2'),
+ owner = 'root',
+ group = 'root',
+ mode = 0644,
)
self.assertResourceCalled('TemplateConfig', '/etc/hbase/conf/hadoop-metrics2-hbase.properties',
owner = 'hbase',
http://git-wip-us.apache.org/repos/asf/ambari/blob/bff61b83/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py
index 74b4aa6..4ca74a8 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py
@@ -277,6 +277,17 @@ class TestHBaseMaster(RMFTestCase):
content = InlineTemplate(self.getConfig()['configurations']['hbase-env']['content']),
group = 'hadoop',
)
+ self.assertResourceCalled('Directory', '/etc/security/limits.d',
+ owner = 'root',
+ group = 'root',
+ recursive = True,
+ )
+ self.assertResourceCalled('File', '/etc/security/limits.d/hbase.conf',
+ content = Template('hbase.conf.j2'),
+ owner = 'root',
+ group = 'root',
+ mode = 0644,
+ )
self.assertResourceCalled('TemplateConfig', '/etc/hbase/conf/hadoop-metrics2-hbase.properties',
owner = 'hbase',
template_tag = 'GANGLIA-MASTER',
@@ -393,6 +404,17 @@ class TestHBaseMaster(RMFTestCase):
content = InlineTemplate(self.getConfig()['configurations']['hbase-env']['content']),
group = 'hadoop',
)
+ self.assertResourceCalled('Directory', '/etc/security/limits.d',
+ owner = 'root',
+ group = 'root',
+ recursive = True,
+ )
+ self.assertResourceCalled('File', '/etc/security/limits.d/hbase.conf',
+ content = Template('hbase.conf.j2'),
+ owner = 'root',
+ group = 'root',
+ mode = 0644,
+ )
self.assertResourceCalled('TemplateConfig', '/etc/hbase/conf/hadoop-metrics2-hbase.properties',
owner = 'hbase',
template_tag = 'GANGLIA-MASTER',
@@ -524,7 +546,17 @@ class TestHBaseMaster(RMFTestCase):
content = InlineTemplate(self.getConfig()['configurations']['hbase-env']['content']),
group = 'hadoop'
)
-
+ self.assertResourceCalled('Directory', '/etc/security/limits.d',
+ owner = 'root',
+ group = 'root',
+ recursive = True,
+ )
+ self.assertResourceCalled('File', '/etc/security/limits.d/hbase.conf',
+ content = Template('hbase.conf.j2'),
+ owner = 'root',
+ group = 'root',
+ mode = 0644,
+ )
self.assertResourceCalled('TemplateConfig', '/usr/hdp/current/hbase-master/conf/hadoop-metrics2-hbase.properties',
owner = 'hbase',
template_tag = 'GANGLIA-MASTER')
http://git-wip-us.apache.org/repos/asf/ambari/blob/bff61b83/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py
index fa134b4..e22584a 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py
@@ -177,6 +177,17 @@ class TestHbaseRegionServer(RMFTestCase):
content = InlineTemplate(self.getConfig()['configurations']['hbase-env']['content']),
group = 'hadoop',
)
+ self.assertResourceCalled('Directory', '/etc/security/limits.d',
+ owner = 'root',
+ group = 'root',
+ recursive = True,
+ )
+ self.assertResourceCalled('File', '/etc/security/limits.d/hbase.conf',
+ content = Template('hbase.conf.j2'),
+ owner = 'root',
+ group = 'root',
+ mode = 0644,
+ )
self.assertResourceCalled('TemplateConfig', '/etc/hbase/conf/hadoop-metrics2-hbase.properties',
owner = 'hbase',
template_tag = 'GANGLIA-RS',
@@ -254,6 +265,17 @@ class TestHbaseRegionServer(RMFTestCase):
content = InlineTemplate(self.getConfig()['configurations']['hbase-env']['content']),
group = 'hadoop',
)
+ self.assertResourceCalled('Directory', '/etc/security/limits.d',
+ owner = 'root',
+ group = 'root',
+ recursive = True,
+ )
+ self.assertResourceCalled('File', '/etc/security/limits.d/hbase.conf',
+ content = Template('hbase.conf.j2'),
+ owner = 'root',
+ group = 'root',
+ mode = 0644,
+ )
self.assertResourceCalled('TemplateConfig', '/etc/hbase/conf/hadoop-metrics2-hbase.properties',
owner = 'hbase',
template_tag = 'GANGLIA-RS',
@@ -346,6 +368,17 @@ class TestHbaseRegionServer(RMFTestCase):
content = InlineTemplate(self.getConfig()['configurations']['hbase-env']['content']),
group = 'hadoop'
)
+ self.assertResourceCalled('Directory', '/etc/security/limits.d',
+ owner = 'root',
+ group = 'root',
+ recursive = True,
+ )
+ self.assertResourceCalled('File', '/etc/security/limits.d/hbase.conf',
+ content = Template('hbase.conf.j2'),
+ owner = 'root',
+ group = 'root',
+ mode = 0644,
+ )
self.assertResourceCalled('TemplateConfig', '/usr/hdp/current/hbase-regionserver/conf/hadoop-metrics2-hbase.properties',
owner = 'hbase',
http://git-wip-us.apache.org/repos/asf/ambari/blob/bff61b83/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_phoenix_queryserver.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_phoenix_queryserver.py b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_phoenix_queryserver.py
index 51a9edc..2cb535c 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_phoenix_queryserver.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_phoenix_queryserver.py
@@ -271,6 +271,17 @@ class TestPhoenixQueryServer(RMFTestCase):
self.getConfig()['configurations']['hbase-env']['content']),
group = 'hadoop',
)
+ self.assertResourceCalled('Directory', '/etc/security/limits.d',
+ owner = 'root',
+ group = 'root',
+ recursive = True,
+ )
+ self.assertResourceCalled('File', '/etc/security/limits.d/hbase.conf',
+ content = Template('hbase.conf.j2'),
+ owner = 'root',
+ group = 'root',
+ mode = 0644,
+ )
self.assertResourceCalled('TemplateConfig',
'/usr/hdp/current/hbase-regionserver/conf/hadoop-metrics2-hbase.properties',
owner = 'hbase',
@@ -361,6 +372,17 @@ class TestPhoenixQueryServer(RMFTestCase):
self.getConfig()['configurations']['hbase-env']['content']),
group = 'hadoop',
)
+ self.assertResourceCalled('Directory', '/etc/security/limits.d',
+ owner = 'root',
+ group = 'root',
+ recursive = True,
+ )
+ self.assertResourceCalled('File', '/etc/security/limits.d/hbase.conf',
+ content = Template('hbase.conf.j2'),
+ owner = 'root',
+ group = 'root',
+ mode = 0644,
+ )
self.assertResourceCalled('TemplateConfig',
'/usr/hdp/current/hbase-regionserver/conf/hadoop-metrics2-hbase.properties',
owner = 'hbase',
@@ -415,4 +437,4 @@ class TestPhoenixQueryServer(RMFTestCase):
)
self.assertResourceCalled('Execute', ('ambari-python-wrap', '/usr/bin/hdp-select', 'set', 'phoenix-server', '2.3.0.0-1234'), sudo=True)
- self.assertNoMoreResources()
\ No newline at end of file
+ self.assertNoMoreResources()