You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jo...@apache.org on 2014/08/28 19:49:48 UTC
[13/45] git commit: AMBARI-7047. BE: Config validation type of values
less than recommended should be WARNs
AMBARI-7047. BE: Config validation type of values less than recommended should be WARNs
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e007d7f4
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e007d7f4
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e007d7f4
Branch: refs/heads/branch-alerts-dev
Commit: e007d7f4fac0bbef2e753e7004e368a0f6329212
Parents: 40e3f0f
Author: Srimanth Gunturi <sg...@hortonworks.com>
Authored: Wed Aug 27 13:51:39 2014 -0700
Committer: Srimanth Gunturi <sg...@hortonworks.com>
Committed: Wed Aug 27 13:51:39 2014 -0700
----------------------------------------------------------------------
.../stacks/HDP/2.0.6/services/stack_advisor.py | 56 +++++++++++---------
.../stacks/2.0.6/common/test_stack_advisor.py | 36 ++++++++++---
2 files changed, 60 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/e007d7f4/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index ad0dcc0..083286e 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -415,58 +415,66 @@ class HDP206StackAdvisor(StackAdvisor):
"YARN": ["yarn-site", self.validateYARNConfigurations]
}.get(serviceName, None)
- def toConfigurationValidationErrors(self, items, siteName):
+ def toConfigurationValidationProblems(self, validationProblems, siteName):
result = []
- for item in items:
- if item["message"] is not None:
- error = { "type": 'configuration', "level": 'ERROR', "message": item["message"], "config-type": siteName, "config-name": item["config-name"] }
- result.append(error)
+ for validationProblem in validationProblems:
+ validationItem = validationProblem.get("item", None)
+ if validationItem is not None:
+ problem = { "type": 'configuration', "level": validationItem["level"], "message": validationItem["message"],
+ "config-type": siteName, "config-name": validationProblem["config-name"] }
+ result.append(problem)
return result
+ def getWarnItem(self, message):
+ return {"level": "WARN", "message": message}
+
+ def getErrorItem(self, message):
+ return {"level": "ERROR", "message": message}
+
def validatorLessThenDefaultValue(self, properties, recommendedDefaults, propertyName):
if not propertyName in properties:
- return "Value should be set"
+ return self.getErrorItem("Value should be set")
value = to_number(properties[propertyName])
if value is None:
- return "Value should be integer"
+ return self.getErrorItem("Value should be integer")
defaultValue = to_number(recommendedDefaults[propertyName])
if defaultValue is None:
return None
if value < defaultValue:
- return "Value is less than the recommended default of {0}".format(defaultValue)
+ return self.getWarnItem("Value is less than the recommended default of {0}".format(defaultValue))
return None
def validateXmxValue(self, properties, recommendedDefaults, propertyName):
if not propertyName in properties:
- return "Value should be set"
+ return self.getErrorItem("Value should be set")
value = properties[propertyName]
defaultValue = recommendedDefaults[propertyName]
if defaultValue is None:
- return "Config's default value can't be null or undefined"
+ return self.getErrorItem("Config's default value can't be null or undefined")
if not checkXmxValueFormat(value):
- return 'Invalid value format'
+ return self.getErrorItem('Invalid value format')
valueInt = formatXmxSizeToBytes(getXmxSize(value))
defaultValueXmx = getXmxSize(defaultValue)
defaultValueInt = formatXmxSizeToBytes(defaultValueXmx)
if valueInt < defaultValueInt:
- return "Value is less than the recommended default of -Xmx" + defaultValueXmx
+ return self.getWarnItem("Value is less than the recommended default of -Xmx" + defaultValueXmx)
return None
def validateMapReduce2Configurations(self, properties, recommendedDefaults):
- validationItems = [ {"config-name": 'mapreduce.map.java.opts', "message": self.validateXmxValue(properties, recommendedDefaults, 'mapreduce.map.java.opts')},
- {"config-name": 'mapreduce.reduce.java.opts', "message": self.validateXmxValue(properties, recommendedDefaults, 'mapreduce.reduce.java.opts')},
- {"config-name": 'mapreduce.task.io.sort.mb', "message": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'mapreduce.task.io.sort.mb')},
- {"config-name": 'mapreduce.map.memory.mb', "message": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'mapreduce.map.memory.mb')},
- {"config-name": 'mapreduce.reduce.memory.mb', "message": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'mapreduce.reduce.memory.mb')},
- {"config-name": 'yarn.app.mapreduce.am.resource.mb', "message": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.app.mapreduce.am.resource.mb')},
- {"config-name": 'yarn.app.mapreduce.am.command-opts', "message": self.validateXmxValue(properties, recommendedDefaults, 'yarn.app.mapreduce.am.command-opts')} ]
- return self.toConfigurationValidationErrors(validationItems, "mapred-site")
+ validationItems = [ {"config-name": 'mapreduce.map.java.opts', "item": self.validateXmxValue(properties, recommendedDefaults, 'mapreduce.map.java.opts')},
+ {"config-name": 'mapreduce.reduce.java.opts', "item": self.validateXmxValue(properties, recommendedDefaults, 'mapreduce.reduce.java.opts')},
+ {"config-name": 'mapreduce.task.io.sort.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'mapreduce.task.io.sort.mb')},
+ {"config-name": 'mapreduce.map.memory.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'mapreduce.map.memory.mb')},
+ {"config-name": 'mapreduce.reduce.memory.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'mapreduce.reduce.memory.mb')},
+ {"config-name": 'yarn.app.mapreduce.am.resource.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.app.mapreduce.am.resource.mb')},
+ {"config-name": 'yarn.app.mapreduce.am.command-opts', "item": self.validateXmxValue(properties, recommendedDefaults, 'yarn.app.mapreduce.am.command-opts')} ]
+ return self.toConfigurationValidationProblems(validationItems, "mapred-site")
def validateYARNConfigurations(self, properties, recommendedDefaults):
- validationItems = [ {"config-name": 'yarn.nodemanager.resource.memory-mb', "message": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.nodemanager.resource.memory-mb')},
- {"config-name": 'yarn.scheduler.minimum-allocation-mb', "message": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.scheduler.minimum-allocation-mb')},
- {"config-name": 'yarn.scheduler.maximum-allocation-mb', "message": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.scheduler.maximum-allocation-mb')} ]
- return self.toConfigurationValidationErrors(validationItems, "yarn-site")
+ validationItems = [ {"config-name": 'yarn.nodemanager.resource.memory-mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.nodemanager.resource.memory-mb')},
+ {"config-name": 'yarn.scheduler.minimum-allocation-mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.scheduler.minimum-allocation-mb')},
+ {"config-name": 'yarn.scheduler.maximum-allocation-mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.scheduler.maximum-allocation-mb')} ]
+ return self.toConfigurationValidationProblems(validationItems, "yarn-site")
# Validation helper methods
http://git-wip-us.apache.org/repos/asf/ambari/blob/e007d7f4/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
index d32fa56..898797a 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
@@ -99,9 +99,9 @@ class TestHDP206StackAdvisor(TestCase):
result = self.stackAdvisor.validateComponentLayout(services, hosts)
expectedItems = [
- {"message": "NameNode and Secondary NameNode cannot be hosted on same machine", "host": "host1"},
- {"message": "NameNode and Secondary NameNode cannot be hosted on same machine", "host": "host1"},
- {"message": "Host is not used", "host": "host2"}
+ {"message": "NameNode and Secondary NameNode cannot be hosted on same machine", "level": "WARN", "host": "host1"},
+ {"message": "NameNode and Secondary NameNode cannot be hosted on same machine", "level": "WARN", "host": "host1"},
+ {"message": "Host is not used", "level": "ERROR", "host": "host2"}
]
self.assertValidationResult(expectedItems, result)
@@ -120,7 +120,27 @@ class TestHDP206StackAdvisor(TestCase):
result = self.stackAdvisor.validateComponentLayout(services, hosts)
expectedItems = [
- {"message": "Cardinality violation, cardinality=ALL, hosts count=1"}
+ {"message": "Cardinality violation, cardinality=ALL, hosts count=1", "level": "ERROR"}
+ ]
+ self.assertValidationResult(expectedItems, result)
+
+ def test_validationWarnMessagesIfLessThanDefault(self):
+ servicesInfo = [
+ {
+ "name": "YARN",
+ "components": []
+ }
+ ]
+ services = self.prepareServices(servicesInfo)
+ services["configurations"] = {"yarn-site":{"properties":{"yarn.nodemanager.resource.memory-mb": "0",
+ "yarn.scheduler.minimum-allocation-mb": "str"}}}
+ hosts = self.prepareHosts([])
+ result = self.stackAdvisor.validateConfigurations(services, hosts)
+
+ expectedItems = [
+ {"message": "Value is less than the recommended default of 2046", "level": "WARN"},
+ {"message": "Value should be integer", "level": "ERROR"},
+ {"message": "Value should be set", "level": "ERROR"}
]
self.assertValidationResult(expectedItems, result)
@@ -139,7 +159,7 @@ class TestHDP206StackAdvisor(TestCase):
result = self.stackAdvisor.validateComponentLayout(services, hosts)
expectedItems = [
- {"message": "Host is not used", "host": "host1"}
+ {"message": "Host is not used", "host": "host1", "level": "ERROR"}
]
self.assertValidationResult(expectedItems, result)
@@ -157,7 +177,7 @@ class TestHDP206StackAdvisor(TestCase):
result = self.stackAdvisor.validateComponentLayout(services, hosts)
expectedItems = [
- {"message": "Cardinality violation, cardinality=0-1, hosts count=2"}
+ {"message": "Cardinality violation, cardinality=0-1, hosts count=2", "level": "ERROR"}
]
self.assertValidationResult(expectedItems, result)
@@ -175,7 +195,7 @@ class TestHDP206StackAdvisor(TestCase):
result = self.stackAdvisor.validateComponentLayout(services, hosts)
expectedItems = [
- {"message": "Host is not used", "host": "host2"}
+ {"message": "Host is not used", "host": "host2", "level": "ERROR"}
]
self.assertValidationResult(expectedItems, result)
@@ -244,7 +264,7 @@ class TestHDP206StackAdvisor(TestCase):
def assertValidationResult(self, expectedItems, result):
actualItems = []
for item in result["items"]:
- next = { "message": item["message"] }
+ next = {"message": item["message"], "level": item["level"]}
try:
next["host"] = item["host"]
except KeyError, err: