You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sr...@apache.org on 2015/04/30 01:46:08 UTC
ambari git commit: AMBARI-10837. HDFS Review: Multiple recommendation
API updates for HDFS configs (mpapirkovskyy via srimanth)
Repository: ambari
Updated Branches:
refs/heads/trunk e866f0426 -> 54be9c4e6
AMBARI-10837. HDFS Review: Multiple recommendation API updates for HDFS configs (mpapirkovskyy via srimanth)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/54be9c4e
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/54be9c4e
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/54be9c4e
Branch: refs/heads/trunk
Commit: 54be9c4e67a035a4316ff8784c5686fd84f04141
Parents: e866f04
Author: Srimanth Gunturi <sg...@hortonworks.com>
Authored: Wed Apr 29 16:45:44 2015 -0700
Committer: Srimanth Gunturi <sg...@hortonworks.com>
Committed: Wed Apr 29 16:45:49 2015 -0700
----------------------------------------------------------------------
.../HDFS/2.1.0.2.0/configuration/hadoop-env.xml | 12 +++++++
.../HDFS/2.1.0.2.0/configuration/hdfs-site.xml | 18 +++-------
.../stacks/HDP/2.0.6/services/stack_advisor.py | 9 +++++
.../HDP/2.2/services/HDFS/themes/theme.json | 7 +++-
.../stacks/HDP/2.2/services/stack_advisor.py | 37 ++++++++++++--------
.../stacks/2.2/common/test_stack_advisor.py | 22 ++++++------
6 files changed, 65 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/54be9c4e/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hadoop-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hadoop-env.xml b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hadoop-env.xml
index 99e1130..5d363b5 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hadoop-env.xml
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hadoop-env.xml
@@ -59,6 +59,12 @@
<value>200</value>
<description>NameNode new generation size</description>
<display-name>NameNode new generation size</display-name>
+ <depends-on>
+ <property>
+ <type>hadoop-env</type>
+ <name>namenode_heapsize</name>
+ </property>
+ </depends-on>
<value-attributes>
<type>int</type>
<minimum>0</minimum>
@@ -72,6 +78,12 @@
<value>200</value>
<description>NameNode maximum new generation size</description>
<display-name>NameNode maximum new generation size</display-name>
+ <depends-on>
+ <property>
+ <type>hadoop-env</type>
+ <name>namenode_heapsize</name>
+ </property>
+ </depends-on>
<value-attributes>
<type>int</type>
<minimum>0</minimum>
http://git-wip-us.apache.org/repos/asf/ambari/blob/54be9c4e/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hdfs-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hdfs-site.xml b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hdfs-site.xml
index 0108b27..68bd0f7 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hdfs-site.xml
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hdfs-site.xml
@@ -167,7 +167,7 @@
<property>
<name>dfs.namenode.safemode.threshold-pct</name>
- <value>1.0f</value>
+ <value>0.999</value>
<description>
Specifies the percentage of blocks that should satisfy
the minimal replication requirement defined by dfs.namenode.replication.min.
@@ -176,18 +176,10 @@
</description>
<display-name>Minimum replicated blocks %ge</display-name>
<value-attributes>
- <type>value-list</type>
- <entries>
- <entry>
- <value>0.99</value>
- <label>0.99f</label>
- </entry>
- <entry>
- <value>1.0</value>
- <label>1.0f</label>
- </entry>
- </entries>
- <selection-cardinality>1</selection-cardinality>
+ <type>float</type>
+ <minimum>0.990</minimum>
+ <maximum>1.000</maximum>
+ <increment-step>0.001</increment-step>
</value-attributes>
</property>
http://git-wip-us.apache.org/repos/asf/ambari/blob/54be9c4e/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index afefa5d..32678a8 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -224,6 +224,15 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
return componentHosts[0]
return None
+ def getHostComponentsByCategories(self, hostname, categories, services, hosts):
+ components = []
+ if services is not None and hosts is not None:
+ for service in services["services"]:
+ components.extend([componentEntry for componentEntry in service["components"]
+ if componentEntry["StackServiceComponents"]["component_category"] in categories
+ and hostname in componentEntry["StackServiceComponents"]["hostnames"]])
+ return components
+
def getConfigurationClusterSummary(self, servicesList, hosts, components, services):
hBaseInstalled = False
http://git-wip-us.apache.org/repos/asf/ambari/blob/54be9c4e/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/themes/theme.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/themes/theme.json b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/themes/theme.json
index 4ff39eb..389c70c 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/themes/theme.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/themes/theme.json
@@ -137,7 +137,12 @@
{
"config": "hdfs-site/dfs.namenode.safemode.threshold-pct",
"widget": {
- "type": "toggle"
+ "type": "slider",
+ "units": [
+ {
+ "unit-name": "percent"
+ }
+ ]
}
},
{
http://git-wip-us.apache.org/repos/asf/ambari/blob/54be9c4e/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
index c3af09b..005d275 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
@@ -72,8 +72,12 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
def recommendHDFSConfigurations(self, configurations, clusterData, services, hosts):
putHdfsSiteProperty = self.putProperty(configurations, "hdfs-site", services)
putHdfsSiteProperty("dfs.datanode.max.transfer.threads", 16384 if clusterData["hBaseInstalled"] else 4096)
+
dataDirsCount = 1
- if "dfs.datanode.data.dir" in configurations["hdfs-site"]["properties"]:
+ # Use users 'dfs.datanode.data.dir' first
+ if "hdfs-site" in services["configurations"] and "dfs.datanode.data.dir" in services["configurations"]["hdfs-site"]["properties"]:
+ dataDirsCount = len(str(services["configurations"]["hdfs-site"]["properties"]["dfs.datanode.data.dir"]).split(","))
+ elif "dfs.datanode.data.dir" in configurations["hdfs-site"]["properties"]:
dataDirsCount = len(str(configurations["hdfs-site"]["properties"]["dfs.datanode.data.dir"]).split(","))
if dataDirsCount <= 2:
failedVolumesTolerated = 0
@@ -97,31 +101,32 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
if ("RANGER" in servicesList) and (rangerPluginEnabled.lower() == 'Yes'.lower()):
putHdfsSiteProperty("dfs.permissions.enabled",'true')
- putHdfsSiteProperty("dfs.namenode.safemode.threshold-pct", "0.99f" if len(namenodeHosts) > 1 else "1.0f")
+ putHdfsSiteProperty("dfs.namenode.safemode.threshold-pct", "0.999" if len(namenodeHosts) > 1 else "1.000")
putHdfsEnvProperty = self.putProperty(configurations, "hadoop-env", services)
putHdfsEnvPropertyAttribute = self.putPropertyAttribute(configurations, "hadoop-env")
putHdfsEnvProperty('namenode_heapsize', max(int(clusterData['totalAvailableRam'] / 2), 1024))
- putHdfsEnvProperty('namenode_opt_newsize', max(int(clusterData['totalAvailableRam'] / 8), 128))
- putHdfsEnvProperty('namenode_opt_maxnewsize', max(int(clusterData['totalAvailableRam'] / 8), 256))
- nn_max_heapsize=None
+ nn_heapsize_limit = None
if (namenodeHosts is not None and len(namenodeHosts) > 0):
if len(namenodeHosts) > 1:
nn_max_heapsize = min(int(namenodeHosts[0]["Hosts"]["total_mem"]), int(namenodeHosts[1]["Hosts"]["total_mem"])) / 1024
+ masters_at_host = max(self.getHostComponentsByCategories(namenodeHosts[0]["Hosts"]["host_name"], ["MASTER"], services, hosts),
+ self.getHostComponentsByCategories(namenodeHosts[1]["Hosts"]["host_name"], ["MASTER"], services, hosts))
else:
nn_max_heapsize = int(namenodeHosts[0]["Hosts"]["total_mem"] / 1024) # total_mem in kb
+ masters_at_host = self.getHostComponentsByCategories(namenodeHosts[0]["Hosts"]["host_name"], ["MASTER"], services, hosts)
putHdfsEnvPropertyAttribute('namenode_heapsize', 'maximum', max(nn_max_heapsize, 1024))
- nn_heapsize = nn_max_heapsize
- nn_heapsize -= clusterData["reservedRam"]
- if clusterData["hBaseInstalled"]:
- nn_heapsize -= clusterData["hbaseRam"]
- putHdfsEnvProperty('namenode_heapsize', max(int(nn_heapsize / 2), 1024))
- putHdfsEnvProperty('namenode_opt_newsize', max(int(nn_heapsize / 8), 128))
- putHdfsEnvProperty('namenode_opt_maxnewsize', max(int(nn_heapsize / 8), 256))
+ nn_heapsize_limit = nn_max_heapsize
+ nn_heapsize_limit -= clusterData["reservedRam"]
+ if len(masters_at_host) > 1:
+ nn_heapsize_limit = int(nn_heapsize_limit/2)
+
+ putHdfsEnvProperty('namenode_heapsize', max(nn_heapsize_limit, 1024))
+
datanodeHosts = self.getHostsWithComponent("HDFS", "DATANODE", services, hosts)
if datanodeHosts is not None and len(datanodeHosts) > 0:
@@ -163,13 +168,15 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
nn_memory_config = nn_memory_configs[index]
#override with new values if applicable
- if nn_max_heapsize is not None and nn_memory_config['nn_heap'] <= nn_max_heapsize:
+ if nn_heapsize_limit is not None and nn_memory_config['nn_heap'] <= nn_heapsize_limit:
putHdfsEnvProperty('namenode_heapsize', nn_memory_config['nn_heap'])
- putHdfsEnvProperty('namenode_opt_newsize', nn_memory_config['nn_opt'])
- putHdfsEnvProperty('namenode_opt_maxnewsize', nn_memory_config['nn_opt'])
putHdfsEnvPropertyAttribute('dtnode_heapsize', 'maximum', int(min_datanode_ram_kb/1024))
+ nn_heapsize = int(configurations["hadoop-env"]["properties"]["namenode_heapsize"])
+ putHdfsEnvProperty('namenode_opt_newsize', max(int(nn_heapsize / 8), 128))
+ putHdfsEnvProperty('namenode_opt_maxnewsize', max(int(nn_heapsize / 8), 128))
+
putHdfsSitePropertyAttribute = self.putPropertyAttribute(configurations, "hdfs-site")
putHdfsSitePropertyAttribute('dfs.datanode.failed.volumes.tolerated', 'maximum', dataDirsCount)
http://git-wip-us.apache.org/repos/asf/ambari/blob/54be9c4e/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
index 53439bf..bfc35f9 100644
--- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
@@ -25,7 +25,7 @@ class TestHDP22StackAdvisor(TestCase):
def setUp(self):
import imp
-
+ self.maxDiff = None
self.testDirectory = os.path.dirname(os.path.abspath(__file__))
stackAdvisorPath = os.path.join(self.testDirectory, '../../../../../main/resources/stacks/stack_advisor.py')
hdp206StackAdvisorPath = os.path.join(self.testDirectory, '../../../../../main/resources/stacks/HDP/2.0.6/services/stack_advisor.py')
@@ -1500,7 +1500,7 @@ class TestHDP22StackAdvisor(TestCase):
clusterData = {
"totalAvailableRam": 2048,
"hBaseInstalled": True,
- "hbaseRam": 111,
+ "hbaseRam": 112,
"reservedRam": 128
}
expected = {
@@ -1518,7 +1518,7 @@ class TestHDP22StackAdvisor(TestCase):
'hdfs-site': {
'properties': {
'dfs.datanode.max.transfer.threads': '16384',
- 'dfs.namenode.safemode.threshold-pct': '1.0f',
+ 'dfs.namenode.safemode.threshold-pct': '1.000',
'dfs.datanode.failed.volumes.tolerated': '1',
'dfs.namenode.handler.count': '25',
'dfs.datanode.data.dir': '/path/1,/path/2,/path/3,/path/4'
@@ -1680,8 +1680,8 @@ class TestHDP22StackAdvisor(TestCase):
)
self.stackAdvisor.recommendHDFSConfigurations(configurations, clusterData, services, hosts)
self.assertEquals(configurations["hadoop-env"]["properties"]["namenode_heapsize"], "3072")
- self.assertEquals(configurations["hadoop-env"]["properties"]["namenode_opt_maxnewsize"], "512")
- self.assertEquals(configurations["hadoop-env"]["properties"]["namenode_opt_maxnewsize"], "512")
+ self.assertEquals(configurations["hadoop-env"]["properties"]["namenode_opt_maxnewsize"], "384")
+ self.assertEquals(configurations["hadoop-env"]["properties"]["namenode_opt_maxnewsize"], "384")
# Test 2 - add more datanodes
for i in xrange(11,30):
hostname = "datanode" + `i`
@@ -1705,8 +1705,8 @@ class TestHDP22StackAdvisor(TestCase):
configurations["hdfs-site"]["properties"]["dfs.datanode.data.dir"] = "/path1,/path2,/path3,/path4"
self.stackAdvisor.recommendHDFSConfigurations(configurations, clusterData, services, hosts)
self.assertEquals(configurations["hadoop-env"]["properties"]["namenode_heapsize"], "9984")
- self.assertEquals(configurations["hadoop-env"]["properties"]["namenode_opt_maxnewsize"], "1280")
- self.assertEquals(configurations["hadoop-env"]["properties"]["namenode_opt_maxnewsize"], "1280")
+ self.assertEquals(configurations["hadoop-env"]["properties"]["namenode_opt_maxnewsize"], "1248")
+ self.assertEquals(configurations["hadoop-env"]["properties"]["namenode_opt_maxnewsize"], "1248")
# Test 3 - more datanodes than host can handle
for i in xrange(31, 90):
hostname = "datanode" + `i`
@@ -1727,10 +1727,10 @@ class TestHDP22StackAdvisor(TestCase):
}
)
self.stackAdvisor.recommendHDFSConfigurations(configurations, clusterData, services, hosts)
- self.assertEquals(configurations["hadoop-env"]["properties"]["namenode_heapsize"], "5000")
- self.assertEquals(configurations["hadoop-env"]["properties"]["namenode_opt_maxnewsize"], "1250")
- self.assertEquals(configurations["hadoop-env"]["properties"]["namenode_opt_maxnewsize"], "1250")
-
+ self.assertEquals(configurations["hadoop-env"]["properties"]["namenode_heapsize"], "10112")
+ self.assertEquals(configurations["hadoop-env"]["properties"]["namenode_opt_maxnewsize"], "1264")
+ self.assertEquals(configurations["hadoop-env"]["properties"]["namenode_opt_maxnewsize"], "1264")
+
# Test 4 - KMS empty test from previous call
self.assertTrue("dfs.encryption.key.provider.uri" not in configurations["hdfs-site"]["properties"])