You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ds...@apache.org on 2015/04/23 20:37:46 UTC
ambari git commit: AMBARI-10700 Exception while initialization of
collectors (dsen)
Repository: ambari
Updated Branches:
refs/heads/trunk a2c86f2a2 -> 9d46a9837
AMBARI-10700 Exception while initialization of collectors (dsen)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9d46a983
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9d46a983
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9d46a983
Branch: refs/heads/trunk
Commit: 9d46a9837da6dc5674880772c54367e539fe3963
Parents: a2c86f2
Author: Dmytro Sen <ds...@apache.org>
Authored: Thu Apr 23 21:28:30 2015 +0300
Committer: Dmytro Sen <ds...@apache.org>
Committed: Thu Apr 23 21:35:36 2015 +0300
----------------------------------------------------------------------
.../stacks/HDP/2.0.6/services/stack_advisor.py | 30 +++++++++++++
.../stacks/HDP/2.2/services/stack_advisor.py | 6 ++-
.../stacks/2.0.6/common/test_stack_advisor.py | 45 ++++++++++++++++++++
.../stacks/2.2/common/test_stack_advisor.py | 4 +-
4 files changed, 82 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/9d46a983/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index 0b54b38..7892c02 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -324,6 +324,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
if siteProperties is not None:
resultItems = method(siteProperties, recommendedDefaults[siteName]["properties"], configurations, services, hosts)
items.extend(resultItems)
+ self.validateMinMax(items, recommendedDefaults, configurations)
return items
def getServiceConfigurationValidators(self):
@@ -337,6 +338,35 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
"ams-site": self.validateAmsSiteConfigurations},
}
+ def validateMinMax(self, items, recommendedDefaults, configurations):
+
+ # required for casting to the proper numeric type before comparison
+ def convertToNumber(number):
+ try:
+ return int(number)
+ except ValueError:
+ return float(number)
+
+ for configName in configurations:
+ validationItems = []
+ if configName in recommendedDefaults and "property_attributes" in recommendedDefaults[configName]:
+ for propertyName in recommendedDefaults[configName]["property_attributes"]:
+ if propertyName in configurations[configName]["properties"]:
+ if "maximum" in recommendedDefaults[configName]["property_attributes"][propertyName] and \
+ propertyName in recommendedDefaults[configName]["properties"]:
+ userValue = convertToNumber(configurations[configName]["properties"][propertyName])
+ maxValue = convertToNumber(recommendedDefaults[configName]["property_attributes"][propertyName]["maximum"])
+ if userValue > maxValue:
+ validationItems.extend([{"config-name": propertyName, "item": self.getWarnItem("Value is greater than the recommended maximum of {0} ".format(maxValue))}])
+ if "minimum" in recommendedDefaults[configName]["property_attributes"][propertyName] and \
+ propertyName in recommendedDefaults[configName]["properties"]:
+ userValue = convertToNumber(configurations[configName]["properties"][propertyName])
+ minValue = convertToNumber(recommendedDefaults[configName]["property_attributes"][propertyName]["minimum"])
+ if userValue < minValue:
+ validationItems.extend([{"config-name": propertyName, "item": self.getWarnItem("Value is less than the recommended minimum of {0} ".format(minValue))}])
+ items.extend(self.toConfigurationValidationProblems(validationItems, configName))
+ pass
+
def validateAmsSiteConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
validationItems = []
http://git-wip-us.apache.org/repos/asf/ambari/blob/9d46a983/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
index 1a8034d..ba05ffa 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
@@ -111,7 +111,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
else:
nn_max_heapsize = int(namenodeHosts[0]["Hosts"]["total_mem"] / 1024) # total_mem in kb
- putHdfsEnvPropertyAttribute('namenode_heapsize', 'maximum', nn_max_heapsize)
+ putHdfsEnvPropertyAttribute('namenode_heapsize', 'maximum', max(nn_max_heapsize, 1024))
nn_heapsize = nn_max_heapsize
nn_heapsize -= clusterData["reservedRam"]
@@ -430,7 +430,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
putMapredProperty('mapreduce.reduce.memory.mb', int(2*int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"])))
putMapredProperty('mapreduce.map.java.opts', "-Xmx" + str(int(0.8*int(configurations["mapred-site"]["properties"]["mapreduce.map.memory.mb"]))) + "m")
putMapredProperty('mapreduce.reduce.java.opts', "-Xmx" + str(int(0.8*int(configurations["mapred-site"]["properties"]["mapreduce.reduce.memory.mb"]))) + "m")
- putMapredProperty('mapreduce.task.io.sort.mb', str(int(0.7*int(configurations["mapred-site"]["properties"]["mapreduce.map.memory.mb"]))))
+ putMapredProperty('mapreduce.task.io.sort.mb', str(min(int(0.7*int(configurations["mapred-site"]["properties"]["mapreduce.map.memory.mb"])), 2047)))
# Property Attributes
putMapredPropertyAttribute = self.putPropertyAttribute(configurations, "mapred-site")
yarnMinAllocationSize = int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"])
@@ -441,6 +441,8 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
putMapredPropertyAttribute("mapreduce.reduce.memory.mb", "minimum", yarnMinAllocationSize)
putMapredPropertyAttribute("yarn.app.mapreduce.am.resource.mb", "maximum", yarnMaxAllocationSize)
putMapredPropertyAttribute("yarn.app.mapreduce.am.resource.mb", "minimum", yarnMinAllocationSize)
+ # Hadoop MR limitation
+ putMapredPropertyAttribute("mapreduce.task.io.sort.mb", "maximum", "2047")
def validateMapReduce2Configurations(self, properties, recommendedDefaults, configurations, services, hosts):
validationItems = [ {"config-name": 'mapreduce.map.java.opts', "item": self.validateXmxValue(properties, recommendedDefaults, 'mapreduce.map.java.opts')},
http://git-wip-us.apache.org/repos/asf/ambari/blob/9d46a983/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
index ef6def1..4167328 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
@@ -207,6 +207,51 @@ class TestHDP206StackAdvisor(TestCase):
]
self.assertValidationResult(expectedItems, result)
+ def test_validationMinMax(self):
+
+ configurations = {
+ "mapred-site": {
+ "properties": {
+ "mapreduce.task.io.sort.mb": "4096",
+ "some_float_value": "0.5",
+ "no_min_or_max_attribute_property": "STRING_VALUE"
+ }
+ }
+ }
+ recommendedDefaults = {
+ "mapred-site": {
+ "properties": {
+ "mapreduce.task.io.sort.mb": "2047",
+ "some_float_value": "0.8",
+ "no_min_or_max_attribute_property": "STRING_VALUE"
+ },
+ "property_attributes": {
+ 'mapreduce.task.io.sort.mb': {'maximum': '2047'},
+ 'some_float_value': {'minimum': '0.8'}
+ }
+ }
+ }
+ items = []
+ self.stackAdvisor.validateMinMax(items, recommendedDefaults, configurations)
+
+ expectedItems = [
+ {
+ 'message': 'Value is greater than the recommended maximum of 2047 ',
+ 'level': 'WARN',
+ 'config-type': 'mapred-site',
+ 'config-name': 'mapreduce.task.io.sort.mb',
+ 'type': 'configuration'
+ },
+ {
+ 'message': 'Value is less than the recommended minimum of 0.8 ',
+ 'level': 'WARN',
+ 'config-type': 'mapred-site',
+ 'config-name': 'some_float_value',
+ 'type': 'configuration'
+ }
+ ]
+ self.assertEquals(expectedItems, items)
+
def test_validationHostIsNotUsedForNonValuableComponent(self):
servicesInfo = [
{
http://git-wip-us.apache.org/repos/asf/ambari/blob/9d46a983/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
index 4c79512..96df408 100644
--- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
@@ -1048,7 +1048,8 @@ class TestHDP22StackAdvisor(TestCase):
"mapreduce.reduce.java.opts": "-Xmx546m",
"yarn.app.mapreduce.am.resource.mb": "682",
"mapreduce.map.java.opts": "-Xmx546m",
- "mapreduce.task.io.sort.mb": "273" }
+ "mapreduce.task.io.sort.mb": "273"
+ }
},
"yarn-site": {
"properties": {
@@ -1081,6 +1082,7 @@ class TestHDP22StackAdvisor(TestCase):
"mapreduce.task.io.sort.mb": "70"
},
"property_attributes": {
+ 'mapreduce.task.io.sort.mb': {'maximum': '2047'},
'yarn.app.mapreduce.am.resource.mb': {'maximum': '1280',
'minimum': '100'},
'mapreduce.map.memory.mb': {'maximum': '1280',