You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sr...@apache.org on 2014/08/18 19:37:28 UTC
git commit: AMBARI-6885. stack_advisor.py from one stack-version
contains another's logic
Repository: ambari
Updated Branches:
refs/heads/trunk eefa2fd57 -> 32caa4359
AMBARI-6885. stack_advisor.py from one stack-version contains another's logic
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/32caa435
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/32caa435
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/32caa435
Branch: refs/heads/trunk
Commit: 32caa4359b7fd777fc7ca9b74e6d8a1212d6f6a0
Parents: eefa2fd
Author: Srimanth Gunturi <sg...@hortonworks.com>
Authored: Fri Aug 15 17:14:51 2014 -0700
Committer: Srimanth Gunturi <sg...@hortonworks.com>
Committed: Mon Aug 18 10:36:04 2014 -0700
----------------------------------------------------------------------
.../stacks/HDP/2.0.6/services/stack_advisor.py | 274 ++++++++-----------
.../stacks/HDP/2.1/services/stack_advisor.py | 68 +++++
.../stacks/2.0.6/common/test_stack_advisor.py | 49 ++--
3 files changed, 211 insertions(+), 180 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/32caa435/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index 025caa2..77c57e2 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -41,15 +41,6 @@ class HDP206StackAdvisor(StackAdvisor):
"services": servicesList,
"recommendations": {
"blueprint": {
- "configurations": {
- "global": {
- "properties": { }
- },
- "core-site": { },
- "hdfs-site": { },
- "yarn-site": { },
- "hbase-site": { }
- },
"host_groups": [ ]
},
"blueprint_cluster_binding": {
@@ -71,7 +62,7 @@ class HDP206StackAdvisor(StackAdvisor):
hostsForComponent = component["StackServiceComponents"]["hostnames"]
else:
availableHosts = hostsList
- if len(hostsList) > 1 and isNotPreferableOnAmbariServerHost(component):
+ if len(hostsList) > 1 and self.isNotPreferableOnAmbariServerHost(component):
availableHosts = [hostName for hostName in hostsList if not isLocalHost(hostName)]
if isMasterWithMultipleInstances(component):
@@ -81,9 +72,9 @@ class HDP206StackAdvisor(StackAdvisor):
hostsCount = len(availableHosts)
hostsForComponent = availableHosts[:hostsCount]
else:
- hostsForComponent = [getHostForComponent(component, availableHosts)]
+ hostsForComponent = [self.getHostForComponent(component, availableHosts)]
else:
- hostsForComponent = [getHostForComponent(component, availableHosts)]
+ hostsForComponent = [self.getHostForComponent(component, availableHosts)]
#extend 'hostsComponentsMap' with 'hostsForComponent'
for hostName in hostsForComponent:
@@ -94,7 +85,7 @@ class HDP206StackAdvisor(StackAdvisor):
#extend 'hostsComponentsMap' with Slave and Client Components
componentsListList = [service["components"] for service in services["services"]]
componentsList = [item for sublist in componentsListList for item in sublist]
- usedHostsListList = [component["StackServiceComponents"]["hostnames"] for component in componentsList if not isNotValuable(component)]
+ usedHostsListList = [component["StackServiceComponents"]["hostnames"] for component in componentsList if not self.isNotValuable(component)]
utilizedHosts = [item for sublist in usedHostsListList for item in sublist]
freeHosts = [hostName for hostName in hostsList if hostName not in utilizedHosts]
@@ -135,6 +126,46 @@ class HDP206StackAdvisor(StackAdvisor):
return recommendations
pass
+ def getHostForComponent(self, component, hostsList):
+ componentName = component["StackServiceComponents"]["component_name"]
+ scheme = self.defineSelectionScheme(componentName)
+
+ if len(hostsList) == 1:
+ return hostsList[0]
+ else:
+ for key in scheme.keys():
+ if isinstance(key, ( int, long )):
+ if len(hostsList) < key:
+ return hostsList[scheme[key]]
+ return hostsList[scheme['else']]
+
+ def defineSelectionScheme(self, componentName):
+ scheme = self.selectionScheme(componentName)
+ if scheme is None:
+ scheme = {"else": 0}
+ return scheme
+
+ def selectionScheme(self, componentName):
+ return {
+ 'NAMENODE': {"else": 0},
+ 'SECONDARY_NAMENODE': {"else": 1},
+ 'HBASE_MASTER': {6: 0, 31: 2, "else": 3},
+
+ 'HISTORYSERVER': {31: 1, "else": 2},
+ 'RESOURCEMANAGER': {31: 1, "else": 2},
+
+ 'OOZIE_SERVER': {6: 1, 31: 2, "else": 3},
+
+ 'HIVE_SERVER': {6: 1, 31: 2, "else": 4},
+ 'HIVE_METASTORE': {6: 1, 31: 2, "else": 4},
+ 'WEBHCAT_SERVER': {6: 1, 31: 2, "else": 4},
+ }.get(componentName, None)
+
+ def isNotPreferableOnAmbariServerHost(self, component):
+ componentName = component["StackServiceComponents"]["component_name"]
+ service = ['GANGLIA_SERVER', 'NAGIOS_SERVER']
+ return componentName in service
+
def validateComponentLayout(self, services, hosts):
"""Returns array of Validation objects about issues with hostnames components assigned to"""
stackName = services["Versions"]["stack_name"]
@@ -195,7 +226,7 @@ class HDP206StackAdvisor(StackAdvisor):
items.append( { "type": 'host-component', "level": 'ERROR', "message": 'Cardinality violation, cardinality={0}, hosts count={1}'.format(cardinality, str(componentHostsCount)), "component-name": str(componentName) } )
# Validating host-usage
- usedHostsListList = [component["StackServiceComponents"]["hostnames"] for component in componentsList if not isNotValuable(component)]
+ usedHostsListList = [component["StackServiceComponents"]["hostnames"] for component in componentsList if not self.isNotValuable(component)]
usedHostsList = [item for sublist in usedHostsListList for item in sublist]
nonUsedHostsList = [item for item in hostsList if item not in usedHostsList]
for host in nonUsedHostsList:
@@ -204,6 +235,11 @@ class HDP206StackAdvisor(StackAdvisor):
return validations
pass
+ def isNotValuable(self, component):
+ componentName = component["StackServiceComponents"]["component_name"]
+ service = ['JOURNALNODE', 'ZKFC', 'GANGLIA_MONITOR']
+ return componentName in service
+
def recommendConfigurations(self, services, hosts):
stackName = services["Versions"]["stack_name"]
stackVersion = services["Versions"]["stack_version"]
@@ -242,9 +278,7 @@ class HDP206StackAdvisor(StackAdvisor):
def recommendServiceConfigurations(self, service):
return {
"YARN": self.recommendYARNConfigurations,
- "MAPREDUCE2": self.recommendMapReduce2Configurations,
- "HIVE": self.recommendHiveConfigurations,
- "OOZIE": self.recommendOozieConfigurations
+ "MAPREDUCE2": self.recommendMapReduce2Configurations
}.get(service, None)
def putProperty(self, config, configType):
@@ -259,15 +293,6 @@ class HDP206StackAdvisor(StackAdvisor):
putYarnProperty('yarn.scheduler.minimum-allocation-mb', clusterData['ramPerContainer'])
putYarnProperty('yarn.scheduler.maximum-allocation-mb', clusterData['containers'] * clusterData['ramPerContainer'])
- def recommendHiveConfigurations(self, configurations, clusterData):
- containerSize = clusterData['mapMemory'] if clusterData['mapMemory'] > 2048 else clusterData['reduceMemory']
- containerSize = min(clusterData['containers'] * clusterData['ramPerContainer'], containerSize)
- putHiveProperty = self.putProperty(configurations, "hive-site")
- putHiveProperty('hive.auto.convert.join.noconditionaltask.size', int(containerSize / 3) * 1048576)
- putHiveProperty('hive.tez.java.opts', "-server -Xmx" + str(int(0.8 * containerSize))
- + "m -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA -XX:+UseParallelGC")
- putHiveProperty('hive.tez.container.size', containerSize)
-
def recommendMapReduce2Configurations(self, configurations, clusterData):
putMapredProperty = self.putProperty(configurations, "mapred-site")
putMapredProperty('yarn.app.mapreduce.am.resource.mb', clusterData['amMemory'])
@@ -278,14 +303,6 @@ class HDP206StackAdvisor(StackAdvisor):
putMapredProperty('mapreduce.reduce.java.opts', "-Xmx" + str(int(0.8 * clusterData['reduceMemory'])) + "m")
putMapredProperty('mapreduce.task.io.sort.mb', int(min(0.4 * clusterData['mapMemory'], 1024)))
- def recommendOozieConfigurations(self, configurations, clusterData):
- if "FALCON_SERVER" in clusterData["components"]:
- putMapredProperty = self.putProperty(configurations, "oozie-site")
- putMapredProperty("oozie.services.ext",
- "org.apache.oozie.service.JMSAccessorService," +
- "org.apache.oozie.service.PartitionDependencyManagerService," +
- "org.apache.oozie.service.HCatAccessorService")
-
def getClusterData(self, servicesList, hosts, components):
hBaseInstalled = False
@@ -380,26 +397,72 @@ class HDP206StackAdvisor(StackAdvisor):
configurations = services["configurations"]
for service in services["services"]:
serviceName = service["StackServices"]["service_name"]
- if serviceName == "MAPREDUCE2":
- mapReduceErrors = validateMapReduce2Configurations(getSiteProperties(configurations, "mapred-site"), recommendedDefaults["mapred-site"]["properties"])
- items.extend(mapReduceErrors)
- elif serviceName == "HIVE":
- hiveErrors = validateHiveConfigurations(getSiteProperties(configurations, "hive-site"), recommendedDefaults["hive-site"]["properties"])
- items.extend(hiveErrors)
- elif serviceName == "STORM":
- oozieErrors = [] #validateStormConfigurations(getSiteProperties(configurations, "storm-site"), recommendedDefaults["storm-site"]["properties"])
- items.extend(oozieErrors)
- elif serviceName == "TEZ":
- tezErrors = validateTezConfigurations(getSiteProperties(configurations, "tez-site"), recommendedDefaults["tez-site"]["properties"])
- items.extend(tezErrors)
- elif serviceName == "YARN":
- yarnErrors = validateYARNConfigurations(getSiteProperties(configurations, "yarn-site"), recommendedDefaults["yarn-site"]["properties"])
- items.extend(yarnErrors)
- else:
- pass
+ validator = self.validateServiceConfigurations(serviceName)
+ if validator is not None:
+ siteName = validator[0]
+ method = validator[1]
+ resultItems = method(getSiteProperties(configurations, siteName), recommendedDefaults[siteName]["properties"])
+ items.extend(resultItems)
return validations
pass
+ def validateServiceConfigurations(self, serviceName):
+ return {
+ "MAPREDUCE2": ["mapred-site", self.validateMapReduce2Configurations],
+ "YARN": ["yarn-site", self.validateYARNConfigurations]
+ }.get(serviceName, None)
+
+ def toConfigurationValidationErrors(self, items, siteName):
+ result = []
+ for item in items:
+ if item["message"] is not None:
+ error = { "type": 'configuration', "level": 'ERROR', "message": item["message"], "config-type": siteName, "config-name": item["config-name"] }
+ result.append(error)
+ return result
+
+ def validatorLessThenDefaultValue(self, properties, recommendedDefaults, propertyName):
+ value = to_number(properties[propertyName])
+ if value is None:
+ return "Value should be integer"
+ defaultValue = to_number(recommendedDefaults[propertyName])
+ if defaultValue is None:
+ return None
+ if value < defaultValue:
+ return "Value is less than the recommended default of {0}".format(defaultValue)
+ return None
+
+ def validateXmxValue(self, properties, recommendedDefaults, propertyName):
+ value = properties[propertyName]
+ defaultValue = recommendedDefaults[propertyName]
+ if defaultValue is None:
+ return "Config's default value can't be null or undefined"
+ if not checkXmxValueFormat(value):
+ return 'Invalid value format'
+ valueInt = formatXmxSizeToBytes(getXmxSize(value))
+ defaultValueXmx = getXmxSize(defaultValue)
+ defaultValueInt = formatXmxSizeToBytes(defaultValueXmx)
+ if valueInt < defaultValueInt:
+ return "Value is less than the recommended default of -Xmx" + defaultValueXmx
+ return None
+
+ def validateMapReduce2Configurations(self, properties, recommendedDefaults):
+ validationItems = [ {"config-name": 'mapreduce.map.java.opts', "message": self.validateXmxValue(properties, recommendedDefaults, 'mapreduce.map.java.opts')},
+ {"config-name": 'mapreduce.reduce.java.opts', "message": self.validateXmxValue(properties, recommendedDefaults, 'mapreduce.reduce.java.opts')},
+ {"config-name": 'mapreduce.task.io.sort.mb', "message": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'mapreduce.task.io.sort.mb')},
+ {"config-name": 'mapreduce.map.memory.mb', "message": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'mapreduce.map.memory.mb')},
+ {"config-name": 'mapreduce.reduce.memory.mb', "message": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'mapreduce.reduce.memory.mb')},
+ {"config-name": 'yarn.app.mapreduce.am.resource.mb', "message": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.app.mapreduce.am.resource.mb')},
+ {"config-name": 'yarn.app.mapreduce.am.command-opts', "message": self.validateXmxValue(properties, recommendedDefaults, 'yarn.app.mapreduce.am.command-opts')} ]
+ return self.toConfigurationValidationErrors(validationItems, "mapred-site")
+
+ def validateYARNConfigurations(self, properties, recommendedDefaults):
+ validationItems = [ {"config-name": 'yarn.nodemanager.resource.memory-mb', "message": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.nodemanager.resource.memory-mb')},
+ {"config-name": 'yarn.scheduler.minimum-allocation-mb', "message": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.scheduler.minimum-allocation-mb')},
+ {"config-name": 'yarn.scheduler.maximum-allocation-mb', "message": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.scheduler.maximum-allocation-mb')} ]
+ return self.toConfigurationValidationErrors(validationItems, "yarn-site")
+
+
+# Validation helper methods
def getSiteProperties(configurations, siteName):
if configurations[siteName] is None:
return {}
@@ -413,39 +476,6 @@ def to_number(s):
except ValueError:
return None
-def toConfigurationValidationErrors(items, siteName):
- result = []
- for item in items:
- if item["message"] is not None:
- error = { "type": 'configuration', "level": 'ERROR', "message": item["message"], "config-type": siteName, "config-name": item["config-name"] }
- result.append(error)
- return result
-
-def validatorLessThenDefaultValue(properties, recommendedDefaults, propertyName):
- value = to_number(properties[propertyName])
- if value is None:
- return "Value should be integer"
- defaultValue = to_number(recommendedDefaults[propertyName])
- if defaultValue is None:
- return None
- if value < defaultValue:
- return "Value is less than the recommended default of {0}".format(defaultValue)
- return None
-
-def validateXmxValue(properties, recommendedDefaults, propertyName):
- value = properties[propertyName]
- defaultValue = recommendedDefaults[propertyName]
- if defaultValue is None:
- return "Config's default value can't be null or undefined"
- if not checkXmxValueFormat(value):
- return 'Invalid value format'
- valueInt = formatXmxSizeToBytes(getXmxSize(value))
- defaultValueXmx = getXmxSize(defaultValue)
- defaultValueInt = formatXmxSizeToBytes(defaultValueXmx)
- if valueInt < defaultValueInt:
- return "Value is less than the recommended default of -Xmx" + defaultValueXmx
- return None
-
def checkXmxValueFormat(value):
p = re.compile('-Xmx(\d+)(b|k|m|g|p|t|B|K|M|G|P|T)?')
matches = p.findall(value)
@@ -477,58 +507,8 @@ def formatXmxSizeToBytes(value):
}[1]
return to_number(value) * m
-def validateMapReduce2Configurations(properties, recommendedDefaults):
- validationItems = [ {"config-name": 'mapreduce.map.java.opts', "message": validateXmxValue(properties, recommendedDefaults, 'mapreduce.map.java.opts')},
- {"config-name": 'mapreduce.reduce.java.opts', "message": validateXmxValue(properties, recommendedDefaults, 'mapreduce.reduce.java.opts')},
- {"config-name": 'mapreduce.task.io.sort.mb', "message": validatorLessThenDefaultValue(properties, recommendedDefaults, 'mapreduce.task.io.sort.mb')},
- {"config-name": 'mapreduce.map.memory.mb', "message": validatorLessThenDefaultValue(properties, recommendedDefaults, 'mapreduce.map.memory.mb')},
- {"config-name": 'mapreduce.reduce.memory.mb', "message": validatorLessThenDefaultValue(properties, recommendedDefaults, 'mapreduce.reduce.memory.mb')},
- {"config-name": 'yarn.app.mapreduce.am.resource.mb', "message": validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.app.mapreduce.am.resource.mb')},
- {"config-name": 'yarn.app.mapreduce.am.command-opts', "message": validateXmxValue(properties, recommendedDefaults, 'yarn.app.mapreduce.am.command-opts')} ]
- return toConfigurationValidationErrors(validationItems, "mapred-site")
-
-def validateHiveConfigurations(properties, recommendedDefaults):
- validationItems = [ {"config-name": 'hive.tez.container.size', "message": validatorLessThenDefaultValue(properties, recommendedDefaults, 'hive.tez.container.size')},
- {"config-name": 'hive.tez.java.opts', "message": validateXmxValue(properties, recommendedDefaults, 'hive.tez.java.opts')},
- {"config-name": 'hive.auto.convert.join.noconditionaltask.size', "message": validatorLessThenDefaultValue(properties, recommendedDefaults, 'hive.auto.convert.join.noconditionaltask.size')} ]
- return toConfigurationValidationErrors(validationItems, "hive-site")
-
-def validateStormConfigurations(properties, recommendedDefaults):
- validationItems = [ {"config-name": 'drpc.childopts', "message": validateXmxValue(properties, recommendedDefaults, 'drpc.childopts')},
- {"config-name": 'ui.childopts', "message": validateXmxValue(properties, recommendedDefaults, 'ui.childopts')},
- {"config-name": 'logviewer.childopts', "message": validateXmxValue(properties, recommendedDefaults, 'logviewer.childopts')} ]
- return toConfigurationValidationErrors(validationItems, "storm-site")
-
-def validateTezConfigurations(properties, recommendedDefaults):
- validationItems = [ {"config-name": 'tez.am.resource.memory.mb', "message": validatorLessThenDefaultValue(properties, recommendedDefaults, 'tez.am.resource.memory.mb')},
- {"config-name": 'tez.am.java.opts', "message": validateXmxValue(properties, recommendedDefaults, 'tez.am.java.opts')} ]
- return toConfigurationValidationErrors(validationItems, "tez-site")
-
-def validateYARNConfigurations(properties, recommendedDefaults):
- validationItems = [ {"config-name": 'yarn.nodemanager.resource.memory-mb', "message": validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.nodemanager.resource.memory-mb')},
- {"config-name": 'yarn.scheduler.minimum-allocation-mb', "message": validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.scheduler.minimum-allocation-mb')},
- {"config-name": 'yarn.scheduler.maximum-allocation-mb', "message": validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.scheduler.maximum-allocation-mb')} ]
- return toConfigurationValidationErrors(validationItems, "yarn-site")
-
-# Helper methods
-def getHostForComponent(component, hostsList):
- componentName = component["StackServiceComponents"]["component_name"]
- scheme = selectionScheme(componentName)
-
- if len(hostsList) == 1:
- return hostsList[0]
- else:
- for key in scheme.keys():
- if isinstance(key, ( int, long )):
- if len(hostsList) < key:
- return hostsList[scheme[key]]
- return hostsList[scheme['else']]
-
-def isNotValuable(component):
- componentName = component["StackServiceComponents"]["component_name"]
- service = ['JOURNALNODE', 'ZKFC', 'APP_TIMELINE_SERVER', 'GANGLIA_MONITOR']
- return componentName in service
+# Recommendation helper methods
def isAlreadyPopulated(component):
if component["StackServiceComponents"]["hostnames"] is not None:
return len(component["StackServiceComponents"]["hostnames"]) > 0
@@ -550,11 +530,6 @@ def isMaster(component):
def isLocalHost(hostName):
return socket.getfqdn(hostName) == socket.getfqdn()
-def isNotPreferableOnAmbariServerHost(component):
- componentName = component["StackServiceComponents"]["component_name"]
- service = ['STORM_UI_SERVER', 'DRPC_SERVER', 'STORM_REST_API', 'NIMBUS', 'GANGLIA_SERVER', 'NAGIOS_SERVER', 'HUE_SERVER']
- return componentName in service
-
def isMasterWithMultipleInstances(component):
componentName = component["StackServiceComponents"]["component_name"]
masters = ['ZOOKEEPER_SERVER', 'HBASE_MASTER']
@@ -572,22 +547,3 @@ def cardinality(componentName):
'HBASE_MASTER': {min: 1},
}.get(componentName, {min:1, max:1})
-def selectionScheme(componentName):
- return {
- 'NAMENODE': {"else": 0},
- 'SECONDARY_NAMENODE': {"else": 1},
- 'HBASE_MASTER': {6: 0, 31: 2, "else": 3},
-
- 'JOBTRACKER': {31: 1, "else": 2},
- 'HISTORYSERVER': {31: 1, "else": 2},
- 'RESOURCEMANAGER': {31: 1, "else": 2},
- 'APP_TIMELINE_SERVER': {31: 1, "else": 2},
-
- 'OOZIE_SERVER': {6: 1, 31: 2, "else": 3},
- 'FALCON_SERVER': {6: 1, 31: 2, "else": 3},
-
- 'HIVE_SERVER': {6: 1, 31: 2, "else": 4},
- 'HIVE_METASTORE': {6: 1, 31: 2, "else": 4},
- 'WEBHCAT_SERVER': {6: 1, 31: 2, "else": 4},
- }.get(componentName, {"else": 0})
-
http://git-wip-us.apache.org/repos/asf/ambari/blob/32caa435/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
index eb2faa1..5d7a3bc 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
@@ -27,11 +27,30 @@ class HDP21StackAdvisor(HDP206StackAdvisor):
calculator = super(HDP21StackAdvisor, self).recommendServiceConfigurations(service)
if calculator is None:
return {
+ "OOZIE": self.recommendOozieConfigurations,
+ "HIVE": self.recommendHiveConfigurations,
"TEZ": self.recommendTezConfigurations
}.get(service, None)
else:
return calculator
+ def recommendOozieConfigurations(self, configurations, clusterData):
+ if "FALCON_SERVER" in clusterData["components"]:
+ putMapredProperty = self.putProperty(configurations, "oozie-site")
+ putMapredProperty("oozie.services.ext",
+ "org.apache.oozie.service.JMSAccessorService," +
+ "org.apache.oozie.service.PartitionDependencyManagerService," +
+ "org.apache.oozie.service.HCatAccessorService")
+
+ def recommendHiveConfigurations(self, configurations, clusterData):
+ containerSize = clusterData['mapMemory'] if clusterData['mapMemory'] > 2048 else clusterData['reduceMemory']
+ containerSize = min(clusterData['containers'] * clusterData['ramPerContainer'], containerSize)
+ putHiveProperty = self.putProperty(configurations, "hive-site")
+ putHiveProperty('hive.auto.convert.join.noconditionaltask.size', int(containerSize / 3) * 1048576)
+ putHiveProperty('hive.tez.java.opts', "-server -Xmx" + str(int(0.8 * containerSize))
+ + "m -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA -XX:+UseParallelGC")
+ putHiveProperty('hive.tez.container.size', containerSize)
+
def recommendTezConfigurations(self, configurations, clusterData):
putTezProperty = self.putProperty(configurations, "tez-site")
putTezProperty("tez.am.resource.memory.mb", clusterData['amMemory'])
@@ -39,3 +58,52 @@ class HDP21StackAdvisor(HDP206StackAdvisor):
"-server -Xmx" + str(int(0.8 * clusterData["amMemory"]))
+ "m -Djava.net.preferIPv4Stack=true -XX:+UseNUMA -XX:+UseParallelGC")
+ def isNotPreferableOnAmbariServerHost(self, component):
+ componentName = component["StackServiceComponents"]["component_name"]
+ service = ['STORM_UI_SERVER', 'DRPC_SERVER', 'STORM_REST_API', 'NIMBUS', 'GANGLIA_SERVER', 'NAGIOS_SERVER']
+ return componentName in service
+
+ def isNotValuable(self, component):
+ componentName = component["StackServiceComponents"]["component_name"]
+ service = ['JOURNALNODE', 'ZKFC', 'GANGLIA_MONITOR', 'APP_TIMELINE_SERVER']
+ return componentName in service
+
+ def selectionScheme(self, componentName):
+ scheme = super(HDP21StackAdvisor, self).selectionScheme(componentName)
+ if scheme is None:
+ return {
+ 'APP_TIMELINE_SERVER': {31: 1, "else": 2},
+ 'FALCON_SERVER': {6: 1, 31: 2, "else": 3}
+ }.get(componentName, None)
+ else:
+ return scheme
+
+ def validateServiceConfigurations(self, serviceName):
+ validator = super(HDP21StackAdvisor, self).validateServiceConfigurations(serviceName)
+ if validator is None:
+ return {
+ "STORM": ["storm-site", self.validateStormConfigurations],
+ "HIVE": ["hive-site", self.validateHiveConfigurations],
+ "TEZ": ["tez-site", self.validateTezConfigurations]
+ }.get(serviceName, None)
+ else:
+ return validator
+
+ def validateHiveConfigurations(self, properties, recommendedDefaults):
+ validationItems = [ {"config-name": 'hive.tez.container.size', "message": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'hive.tez.container.size')},
+ {"config-name": 'hive.tez.java.opts', "message": self.validateXmxValue(properties, recommendedDefaults, 'hive.tez.java.opts')},
+ {"config-name": 'hive.auto.convert.join.noconditionaltask.size', "message": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'hive.auto.convert.join.noconditionaltask.size')} ]
+ return self.toConfigurationValidationErrors(validationItems, "hive-site")
+
+ def validateStormConfigurations(self, properties, recommendedDefaults):
+ validationItems = [ {"config-name": 'drpc.childopts', "message": self.validateXmxValue(properties, recommendedDefaults, 'drpc.childopts')},
+ {"config-name": 'ui.childopts', "message": self.validateXmxValue(properties, recommendedDefaults, 'ui.childopts')},
+ {"config-name": 'logviewer.childopts', "message": self.validateXmxValue(properties, recommendedDefaults, 'logviewer.childopts')} ]
+ return self.toConfigurationValidationErrors(validationItems, "storm-site")
+
+ def validateTezConfigurations(self, properties, recommendedDefaults):
+ validationItems = [ {"config-name": 'tez.am.resource.memory.mb', "message": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'tez.am.resource.memory.mb')},
+ {"config-name": 'tez.am.java.opts', "message": self.validateXmxValue(properties, recommendedDefaults, 'tez.am.java.opts')} ]
+ return self.toConfigurationValidationErrors(validationItems, "tez-site")
+
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/32caa435/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
index 3139a3e..b5bdba6 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
@@ -35,7 +35,7 @@ class TestHDP206StackAdvisor(TestCase):
stack_advisor_impl = imp.load_module('stack_advisor_impl', fp, hdp206StackAdvisorPath, ('.py', 'rb', imp.PY_SOURCE))
clazz = getattr(stack_advisor_impl, hdp206StackAdvisorClassName)
self.stackAdvisor = clazz()
-
+
def test_recommendationCardinalityALL(self):
servicesInfo = [
{
@@ -98,12 +98,12 @@ class TestHDP206StackAdvisor(TestCase):
hosts = self.prepareHosts(["host1", "host2"])
result = self.stackAdvisor.validateComponentLayout(services, hosts)
- expectedMessages = [
- "NameNode and Secondary NameNode cannot be hosted on same machine",
- "NameNode and Secondary NameNode cannot be hosted on same machine",
- "Host is not used"
+ expectedItems = [
+ {"message": "NameNode and Secondary NameNode cannot be hosted on same machine", "host": "host1"},
+ {"message": "NameNode and Secondary NameNode cannot be hosted on same machine", "host": "host1"},
+ {"message": "Host is not used", "host": "host2"}
]
- self.assertValidationMessages(expectedMessages, result)
+ self.assertValidationResult(expectedItems, result)
def test_validationCardinalityALL(self):
servicesInfo = [
@@ -119,10 +119,10 @@ class TestHDP206StackAdvisor(TestCase):
hosts = self.prepareHosts(["host1", "host2"])
result = self.stackAdvisor.validateComponentLayout(services, hosts)
- expectedMessages = [
- "Cardinality violation, cardinality=ALL, hosts count=1"
+ expectedItems = [
+ {"message": "Cardinality violation, cardinality=ALL, hosts count=1"}
]
- self.assertValidationMessages(expectedMessages, result)
+ self.assertValidationResult(expectedItems, result)
def test_validationHostIsNotUsedForNonValuableComponent(self):
servicesInfo = [
@@ -138,10 +138,10 @@ class TestHDP206StackAdvisor(TestCase):
hosts = self.prepareHosts(["host1", "host2"])
result = self.stackAdvisor.validateComponentLayout(services, hosts)
- expectedMessages = [
- "Host is not used"
+ expectedItems = [
+ {"message": "Host is not used", "host": "host1"}
]
- self.assertValidationMessages(expectedMessages, result)
+ self.assertValidationResult(expectedItems, result)
def test_validationCardinality01TwoHostsAssigned(self):
servicesInfo = [
@@ -156,10 +156,10 @@ class TestHDP206StackAdvisor(TestCase):
hosts = self.prepareHosts(["host1", "host2"])
result = self.stackAdvisor.validateComponentLayout(services, hosts)
- expectedMessages = [
- "Cardinality violation, cardinality=0-1, hosts count=2"
+ expectedItems = [
+ {"message": "Cardinality violation, cardinality=0-1, hosts count=2"}
]
- self.assertValidationMessages(expectedMessages, result)
+ self.assertValidationResult(expectedItems, result)
def test_validationHostIsNotUsed(self):
servicesInfo = [
@@ -174,10 +174,10 @@ class TestHDP206StackAdvisor(TestCase):
hosts = self.prepareHosts(["host1", "host2"])
result = self.stackAdvisor.validateComponentLayout(services, hosts)
- expectedMessages = [
- "Host is not used"
+ expectedItems = [
+ {"message": "Host is not used", "host": "host2"}
]
- self.assertValidationMessages(expectedMessages, result)
+ self.assertValidationResult(expectedItems, result)
def prepareHosts(self, hostsNames):
@@ -241,8 +241,15 @@ class TestHDP206StackAdvisor(TestCase):
if not len(l1) == len(l2) or not sorted(l1) == sorted(l2):
raise AssertionError("list1={0}, list2={1}".format(l1, l2))
- def assertValidationMessages(self, expectedMessages, result):
- realMessages = [item["message"] for item in result["items"]]
- self.checkEqual(expectedMessages, realMessages)
+ def assertValidationResult(self, expectedItems, result):
+ actualItems = []
+ for item in result["items"]:
+ next = { "message": item["message"] }
+ try:
+ next["host"] = item["host"]
+ except KeyError, err:
+ pass
+ actualItems.append(next)
+ self.checkEqual(expectedItems, actualItems)