You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by dm...@apache.org on 2015/01/05 15:36:54 UTC
ambari git commit: AMBARI-8985. Configs: NameNode Heapsize stack
advisor (dlysnichenko)
Repository: ambari
Updated Branches:
refs/heads/trunk acf4de184 -> 8a7a4f2e3
AMBARI-8985. Configs: NameNode Heapsize stack advisor (dlysnichenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8a7a4f2e
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8a7a4f2e
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8a7a4f2e
Branch: refs/heads/trunk
Commit: 8a7a4f2e391479db2d0d903d947a21d13fde8ab8
Parents: acf4de1
Author: Lisnichenko Dmitro <dl...@hortonworks.com>
Authored: Mon Jan 5 16:36:04 2015 +0200
Committer: Lisnichenko Dmitro <dl...@hortonworks.com>
Committed: Mon Jan 5 16:36:04 2015 +0200
----------------------------------------------------------------------
.../stacks/HDP/2.0.6/services/stack_advisor.py | 18 ++++++-
.../stacks/HDP/2.2/services/stack_advisor.py | 15 +++++-
.../stacks/2.0.6/common/test_stack_advisor.py | 50 +++++++++++++++++
.../stacks/2.1/common/test_stack_advisor.py | 50 +++++++++++++++++
.../stacks/2.2/common/test_stack_advisor.py | 56 ++++++++++++++++++++
5 files changed, 187 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/8a7a4f2e/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index e88c2c5..9632f05 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -82,7 +82,8 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
def getServiceConfigurationRecommenderDict(self):
return {
"YARN": self.recommendYARNConfigurations,
- "MAPREDUCE2": self.recommendMapReduce2Configurations
+ "MAPREDUCE2": self.recommendMapReduce2Configurations,
+ "HDFS": self.recommendHDFSConfigurations
}
def putProperty(self, config, configType):
@@ -109,6 +110,14 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
putMapredProperty('mapreduce.map.java.opts', "-Xmx" + str(int(round(0.8 * clusterData['mapMemory']))) + "m")
putMapredProperty('mapreduce.reduce.java.opts', "-Xmx" + str(int(round(0.8 * clusterData['reduceMemory']))) + "m")
putMapredProperty('mapreduce.task.io.sort.mb', min(int(round(0.4 * clusterData['mapMemory'])), 1024))
+
+ def recommendHDFSConfigurations(self, configurations, clusterData, services, hosts):
+ putHDFSProperty = self.putProperty(configurations, "hadoop-env")
+ putHDFSProperty('namenode_heapsize', max(int(clusterData['totalAvailableRam'] / 2), 1024))
+ putHDFSProperty = self.putProperty(configurations, "hadoop-env")
+ putHDFSProperty('namenode_opt_newsize', max(int(clusterData['totalAvailableRam'] / 8), 128))
+ putHDFSProperty = self.putProperty(configurations, "hadoop-env")
+ putHDFSProperty('namenode_opt_maxnewsize', max(int(clusterData['totalAvailableRam'] / 8), 256))
def getConfigurationClusterSummary(self, servicesList, hosts, components):
@@ -210,6 +219,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
def getServiceConfigurationValidators(self):
return {
+ "HDFS": {"hadoop-env": self.validateHDFSConfigurationsEnv},
"MAPREDUCE2": {"mapred-site": self.validateMapReduce2Configurations},
"YARN": {"yarn-site": self.validateYARNConfigurations}
}
@@ -293,6 +303,12 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
{"config-name": 'yarn.scheduler.maximum-allocation-mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.scheduler.maximum-allocation-mb')} ]
return self.toConfigurationValidationProblems(validationItems, "yarn-site")
+ def validateHDFSConfigurationsEnv(self, properties, recommendedDefaults, configurations, services, hosts):
+ validationItems = [ {"config-name": 'namenode_heapsize', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'namenode_heapsize')},
+ {"config-name": 'namenode_opt_newsize', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'namenode_opt_newsize')},
+ {"config-name": 'namenode_opt_maxnewsize', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'namenode_opt_maxnewsize')}]
+ return self.toConfigurationValidationProblems(validationItems, "hadoop-env")
+
def getMastersWithMultipleInstances(self):
return ['ZOOKEEPER_SERVER', 'HBASE_MASTER']
http://git-wip-us.apache.org/repos/asf/ambari/blob/8a7a4f2e/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
index fbfc850..44c04a2 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
@@ -39,6 +39,12 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
def recommendHDFSConfigurations(self, configurations, clusterData, services, hosts):
putHdfsPropery = self.putProperty(configurations, "hdfs-site")
putHdfsPropery("dfs.datanode.max.transfer.threads", 16384 if clusterData["hBaseInstalled"] else 4096)
+ putHDFSProperty = self.putProperty(configurations, "hadoop-env")
+ putHDFSProperty('namenode_heapsize', max(int(clusterData['totalAvailableRam'] / 2), 1024))
+ putHDFSProperty = self.putProperty(configurations, "hadoop-env")
+ putHDFSProperty('namenode_opt_newsize', max(int(clusterData['totalAvailableRam'] / 8), 128))
+ putHDFSProperty = self.putProperty(configurations, "hadoop-env")
+ putHDFSProperty('namenode_opt_maxnewsize', max(int(clusterData['totalAvailableRam'] / 8), 256))
def recommendTezConfigurations(self, configurations, clusterData, services, hosts):
putTezProperty = self.putProperty(configurations, "tez-site")
@@ -91,7 +97,8 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
def getServiceConfigurationValidators(self):
parentValidators = super(HDP22StackAdvisor, self).getServiceConfigurationValidators()
childValidators = {
- "HDFS": {"hdfs-site": self.validateHDFSConfigurations},
+ "HDFS": {"hdfs-site": self.validateHDFSConfigurations,
+ "hadoop-env": self.validateHDFSConfigurationsEnv},
"MAPREDUCE2": {"mapred-site": self.validateMapReduce2Configurations},
"AMS": {"ams-hbase-site": self.validateAmsHbaseSiteConfigurations,
"ams-hbase-env": self.validateAmsHbaseEnvConfigurations},
@@ -180,6 +187,12 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
{"config-name": 'yarn.app.mapreduce.am.resource.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.app.mapreduce.am.resource.mb')}]
return self.toConfigurationValidationProblems(validationItems, "mapred-site")
+ def validateHDFSConfigurationsEnv(self, properties, recommendedDefaults, configurations, services, hosts):
+ validationItems = [ {"config-name": 'namenode_heapsize', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'namenode_heapsize')},
+ {"config-name": 'namenode_opt_newsize', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'namenode_opt_newsize')},
+ {"config-name": 'namenode_opt_maxnewsize', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'namenode_opt_maxnewsize')}]
+ return self.toConfigurationValidationProblems(validationItems, "hadoop-env")
+
def validateHDFSConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
# We can not access property hadoop.security.authentication from the
# other config (core-site). That's why we are using another heuristics here
http://git-wip-us.apache.org/repos/asf/ambari/blob/8a7a4f2e/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
index fc5f66b..333f8d4 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
@@ -498,3 +498,53 @@ class TestHDP206StackAdvisor(TestCase):
pass
actualItems.append(next)
self.checkEqual(expectedItems, actualItems)
+
+ def test_recommendHDFSConfigurations(self):
+ configurations = {}
+ clusterData = {
+ "totalAvailableRam": 2048
+ }
+ expected = {
+ 'hadoop-env': {
+ 'properties': {
+ 'namenode_heapsize': '1024',
+ 'namenode_opt_newsize' : '256',
+ 'namenode_opt_maxnewsize' : '256'
+ }
+ }
+ }
+
+ self.stackAdvisor.recommendHDFSConfigurations(configurations, clusterData, '', '')
+ self.assertEquals(configurations, expected)
+
+ def test_validateHDFSConfigurationsEnv(self):
+ configurations = {}
+
+ # 1) ok: namenode_heapsize > recommended
+ recommendedDefaults = {'namenode_heapsize': '1024',
+ 'namenode_opt_newsize' : '256',
+ 'namenode_opt_maxnewsize' : '256'}
+ properties = {'namenode_heapsize': '2048',
+ 'namenode_opt_newsize' : '300',
+ 'namenode_opt_maxnewsize' : '300'}
+ res_expected = []
+
+ res = self.stackAdvisor.validateHDFSConfigurationsEnv(properties, recommendedDefaults, configurations, '', '')
+ self.assertEquals(res, res_expected)
+
+ # 2) fail: namenode_heapsize, namenode_opt_maxnewsize < recommended
+ properties['namenode_heapsize'] = '1022'
+ properties['namenode_opt_maxnewsize'] = '255'
+ res_expected = [{'config-type': 'hadoop-env',
+ 'message': 'Value is less than the recommended default of 1024',
+ 'type': 'configuration',
+ 'config-name': 'namenode_heapsize',
+ 'level': 'WARN'},
+ {'config-name': 'namenode_opt_maxnewsize',
+ 'config-type': 'hadoop-env',
+ 'level': 'WARN',
+ 'message': 'Value is less than the recommended default of 256',
+ 'type': 'configuration'}]
+
+ res = self.stackAdvisor.validateHDFSConfigurationsEnv(properties, recommendedDefaults, configurations, '', '')
+ self.assertEquals(res, res_expected)
http://git-wip-us.apache.org/repos/asf/ambari/blob/8a7a4f2e/ambari-server/src/test/python/stacks/2.1/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.1/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.1/common/test_stack_advisor.py
index cd756ae..790b7d5 100644
--- a/ambari-server/src/test/python/stacks/2.1/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.1/common/test_stack_advisor.py
@@ -159,3 +159,53 @@ class TestHDP21StackAdvisor(TestCase):
self.stackAdvisor.recommendHiveConfigurations(configurations, clusterData, None, None)
self.assertEquals(configurations, expected)
+
+ def test_recommendHDFSConfigurations(self):
+ configurations = {}
+ clusterData = {
+ "totalAvailableRam": 2048
+ }
+ expected = {
+ 'hadoop-env': {
+ 'properties': {
+ 'namenode_heapsize': '1024',
+ 'namenode_opt_newsize' : '256',
+ 'namenode_opt_maxnewsize' : '256'
+ }
+ }
+ }
+
+ self.stackAdvisor.recommendHDFSConfigurations(configurations, clusterData, '', '')
+ self.assertEquals(configurations, expected)
+
+ def test_validateHDFSConfigurationsEnv(self):
+ configurations = {}
+
+ # 1) ok: namenode_heapsize > recommended
+ recommendedDefaults = {'namenode_heapsize': '1024',
+ 'namenode_opt_newsize' : '256',
+ 'namenode_opt_maxnewsize' : '256'}
+ properties = {'namenode_heapsize': '2048',
+ 'namenode_opt_newsize' : '300',
+ 'namenode_opt_maxnewsize' : '300'}
+ res_expected = []
+
+ res = self.stackAdvisor.validateHDFSConfigurationsEnv(properties, recommendedDefaults, configurations, '', '')
+ self.assertEquals(res, res_expected)
+
+ # 2) fail: namenode_heapsize, namenode_opt_maxnewsize < recommended
+ properties['namenode_heapsize'] = '1022'
+ properties['namenode_opt_maxnewsize'] = '255'
+ res_expected = [{'config-type': 'hadoop-env',
+ 'message': 'Value is less than the recommended default of 1024',
+ 'type': 'configuration',
+ 'config-name': 'namenode_heapsize',
+ 'level': 'WARN'},
+ {'config-name': 'namenode_opt_maxnewsize',
+ 'config-type': 'hadoop-env',
+ 'level': 'WARN',
+ 'message': 'Value is less than the recommended default of 256',
+ 'type': 'configuration'}]
+
+ res = self.stackAdvisor.validateHDFSConfigurationsEnv(properties, recommendedDefaults, configurations, '', '')
+ self.assertEquals(res, res_expected)
http://git-wip-us.apache.org/repos/asf/ambari/blob/8a7a4f2e/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
index 4bdd52a..6fab864 100644
--- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
@@ -576,3 +576,59 @@ class TestHDP22StackAdvisor(TestCase):
}
self.stackAdvisor.recommendAmsConfigurations(configurations, clusterData, services, hosts)
self.assertEquals(configurations, expected)
+
+ def test_recommendHDFSConfigurations(self):
+ configurations = {}
+ clusterData = {
+ "totalAvailableRam": 2048,
+ "hBaseInstalled": 111
+ }
+ expected = {
+ 'hadoop-env': {
+ 'properties': {
+ 'namenode_heapsize': '1024',
+ 'namenode_opt_newsize' : '256',
+ 'namenode_opt_maxnewsize' : '256'
+ }
+ },
+ 'hdfs-site': {
+ 'properties': {
+ 'dfs.datanode.max.transfer.threads': '16384'
+ }
+ }
+ }
+
+ self.stackAdvisor.recommendHDFSConfigurations(configurations, clusterData, '', '')
+ self.assertEquals(configurations, expected)
+
+ def test_validateHDFSConfigurationsEnv(self):
+ configurations = {}
+
+ # 1) ok: namenode_heapsize > recommended
+ recommendedDefaults = {'namenode_heapsize': '1024',
+ 'namenode_opt_newsize' : '256',
+ 'namenode_opt_maxnewsize' : '256'}
+ properties = {'namenode_heapsize': '2048',
+ 'namenode_opt_newsize' : '300',
+ 'namenode_opt_maxnewsize' : '300'}
+ res_expected = []
+
+ res = self.stackAdvisor.validateHDFSConfigurationsEnv(properties, recommendedDefaults, configurations, '', '')
+ self.assertEquals(res, res_expected)
+
+ # 2) fail: namenode_heapsize, namenode_opt_maxnewsize < recommended
+ properties['namenode_heapsize'] = '1022'
+ properties['namenode_opt_maxnewsize'] = '255'
+ res_expected = [{'config-type': 'hadoop-env',
+ 'message': 'Value is less than the recommended default of 1024',
+ 'type': 'configuration',
+ 'config-name': 'namenode_heapsize',
+ 'level': 'WARN'},
+ {'config-name': 'namenode_opt_maxnewsize',
+ 'config-type': 'hadoop-env',
+ 'level': 'WARN',
+ 'message': 'Value is less than the recommended default of 256',
+ 'type': 'configuration'}]
+
+ res = self.stackAdvisor.validateHDFSConfigurationsEnv(properties, recommendedDefaults, configurations, '', '')
+ self.assertEquals(res, res_expected)