You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sr...@apache.org on 2014/09/25 22:37:36 UTC
git commit: AMBARI-7464. Make stack_advisor code more human readable
- additional changes (srimanth)
Repository: ambari
Updated Branches:
refs/heads/trunk cb6561206 -> 6c74b724c
AMBARI-7464. Make stack_advisor code more human readable - additional changes (srimanth)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6c74b724
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6c74b724
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6c74b724
Branch: refs/heads/trunk
Commit: 6c74b724c815e7bed7d4ff8c36bad4fd3d4dda85
Parents: cb656120
Author: Srimanth Gunturi <sg...@hortonworks.com>
Authored: Thu Sep 25 12:24:23 2014 -0700
Committer: Srimanth Gunturi <sg...@hortonworks.com>
Committed: Thu Sep 25 13:16:10 2014 -0700
----------------------------------------------------------------------
.../stacks/HDP/1.3.2/services/stack_advisor.py | 10 +--
.../stacks/HDP/2.0.6/services/stack_advisor.py | 10 +--
.../stacks/HDP/2.1/services/stack_advisor.py | 39 +++------
.../src/main/resources/stacks/stack_advisor.py | 90 ++++++++++++--------
4 files changed, 74 insertions(+), 75 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/6c74b724/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/stack_advisor.py
index ba42075..ca7c1a1 100644
--- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/stack_advisor.py
@@ -25,7 +25,7 @@ from stack_advisor import DefaultStackAdvisor
class HDP132StackAdvisor(DefaultStackAdvisor):
- def getLayoutValidationItems(self, services, hosts):
+ def getComponentLayoutValidations(self, services, hosts):
"""Returns array of Validation objects about issues with hostnames components assigned to"""
items = []
@@ -73,7 +73,7 @@ class HDP132StackAdvisor(DefaultStackAdvisor):
items.append( { "type": 'host-component', "level": 'ERROR', "message": 'Cardinality violation, cardinality={0}, hosts count={1}'.format(cardinality, str(componentHostsCount)), "component-name": str(componentName) } )
# Validating host-usage
- usedHostsListList = [component["StackServiceComponents"]["hostnames"] for component in componentsList if not self.isNotValuable(component)]
+ usedHostsListList = [component["StackServiceComponents"]["hostnames"] for component in componentsList if not self.isComponentNotValuable(component)]
usedHostsList = [item for sublist in usedHostsListList for item in sublist]
nonUsedHostsList = [item for item in hostsList if item not in usedHostsList]
for host in nonUsedHostsList:
@@ -81,7 +81,7 @@ class HDP132StackAdvisor(DefaultStackAdvisor):
return items
- def getServiceConfiguratorDict(self):
+ def getServiceConfigurationRecommenderDict(self):
return {
}
@@ -91,7 +91,7 @@ class HDP132StackAdvisor(DefaultStackAdvisor):
config[configType]["properties"][key] = str(value)
return appendProperty
- def getClusterData(self, servicesList, hosts, components):
+ def getConfigurationClusterSummary(self, servicesList, hosts, components):
hBaseInstalled = False
if 'HBASE' in servicesList:
@@ -255,7 +255,7 @@ class HDP132StackAdvisor(DefaultStackAdvisor):
'HBASE_MASTER': {"min": 1},
}
- def selectionSchemes(self):
+ def getComponentLayoutSchemes(self):
return {
'NAMENODE': {"else": 0},
'SECONDARY_NAMENODE': {"else": 1},
http://git-wip-us.apache.org/repos/asf/ambari/blob/6c74b724/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index 8853e13..4057392 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -25,7 +25,7 @@ from stack_advisor import DefaultStackAdvisor
class HDP206StackAdvisor(DefaultStackAdvisor):
- def getLayoutValidationItems(self, services, hosts):
+ def getComponentLayoutValidations(self, services, hosts):
"""Returns array of Validation objects about issues with hostnames components assigned to"""
items = []
@@ -70,7 +70,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
items.append({"type": 'host-component', "level": 'ERROR', "message": message, "component-name": componentName})
# Validating host-usage
- usedHostsListList = [component["StackServiceComponents"]["hostnames"] for component in componentsList if not self.isNotValuable(component)]
+ usedHostsListList = [component["StackServiceComponents"]["hostnames"] for component in componentsList if not self.isComponentNotValuable(component)]
usedHostsList = [item for sublist in usedHostsListList for item in sublist]
nonUsedHostsList = [item for item in hostsList if item not in usedHostsList]
for host in nonUsedHostsList:
@@ -78,7 +78,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
return items
- def getServiceConfiguratorDict(self):
+ def getServiceConfigurationRecommenderDict(self):
return {
"YARN": self.recommendYARNConfigurations,
"MAPREDUCE2": self.recommendMapReduce2Configurations
@@ -106,7 +106,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
putMapredProperty('mapreduce.reduce.java.opts', "-Xmx" + str(int(round(0.8 * clusterData['reduceMemory']))) + "m")
putMapredProperty('mapreduce.task.io.sort.mb', min(int(round(0.4 * clusterData['mapMemory'])), 1024))
- def getClusterData(self, servicesList, hosts, components):
+ def getConfigurationClusterSummary(self, servicesList, hosts, components):
hBaseInstalled = False
if 'HBASE' in servicesList:
@@ -290,7 +290,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
'HBASE_MASTER': {"min": 1},
}
- def selectionSchemes(self):
+ def getComponentLayoutSchemes(self):
return {
'NAMENODE': {"else": 0},
'SECONDARY_NAMENODE': {"else": 1},
http://git-wip-us.apache.org/repos/asf/ambari/blob/6c74b724/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
index 621e74d..f45211f 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
@@ -19,16 +19,15 @@ limitations under the License.
class HDP21StackAdvisor(HDP206StackAdvisor):
- def recommendServiceConfigurations(self, service):
- calculator = super(HDP21StackAdvisor, self).recommendServiceConfigurations(service)
- if calculator is None:
- return {
- "OOZIE": self.recommendOozieConfigurations,
- "HIVE": self.recommendHiveConfigurations,
- "TEZ": self.recommendTezConfigurations
- }.get(service, None)
- else:
- return calculator
+ def getServiceConfigurationRecommenderDict(self, service):
+ parentRecommendConfDict = super(HDP21StackAdvisor, self).getServiceConfigurationRecommenderDict()
+ childRecommendConfDict = {
+ "OOZIE": self.recommendOozieConfigurations,
+ "HIVE": self.recommendHiveConfigurations,
+ "TEZ": self.recommendTezConfigurations
+ }
+ parentRecommendConfDict.update(childRecommendConfDict)
+ return parentRecommendConfDict
def recommendOozieConfigurations(self, configurations, clusterData):
if "FALCON_SERVER" in clusterData["components"]:
@@ -60,24 +59,8 @@ class HDP21StackAdvisor(HDP206StackAdvisor):
def getNotValuableComponents(self):
return ['JOURNALNODE', 'ZKFC', 'GANGLIA_MONITOR', 'APP_TIMELINE_SERVER']
- def selectionSchemes(self):
- return {
- 'NAMENODE': {"else": 0},
- 'SECONDARY_NAMENODE': {"else": 1},
- 'HBASE_MASTER': {6: 0, 31: 2, "else": 3},
-
- 'HISTORYSERVER': {31: 1, "else": 2},
- 'RESOURCEMANAGER': {31: 1, "else": 2},
-
- 'OOZIE_SERVER': {6: 1, 31: 2, "else": 3},
-
- 'HIVE_SERVER': {6: 1, 31: 2, "else": 4},
- 'HIVE_METASTORE': {6: 1, 31: 2, "else": 4},
- 'WEBHCAT_SERVER': {6: 1, 31: 2, "else": 4},
- }
-
- def selectionScheme(self):
- parentSchemes = super(HDP21StackAdvisor, self).selectionSchemes()
+ def getComponentLayoutSchemes(self):
+ parentSchemes = super(HDP21StackAdvisor, self).getComponentLayoutSchemes()
childSchemes = {
'APP_TIMELINE_SERVER': {31: 1, "else": 2},
'FALCON_SERVER': {6: 1, 31: 2, "else": 3}
http://git-wip-us.apache.org/repos/asf/ambari/blob/6c74b724/ambari-server/src/main/resources/stacks/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/stack_advisor.py b/ambari-server/src/main/resources/stacks/stack_advisor.py
index f2b5446..28dc932 100644
--- a/ambari-server/src/main/resources/stacks/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/stack_advisor.py
@@ -300,6 +300,13 @@ class StackAdvisor(object):
class DefaultStackAdvisor(StackAdvisor):
+ """
+ Default stack advisor implementation.
+
+ This implementation is used when a stack-version, or its hierarchy does not
+ have an advisor. Stack-versions can extend this class to provide their own
+ implement
+ """
def recommendComponentLayout(self, services, hosts):
"""Returns Services object with hostnames array populated for components"""
@@ -309,7 +316,7 @@ class DefaultStackAdvisor(StackAdvisor):
hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
- layoutRecommendations = self.provideLayout(services, hosts)
+ layoutRecommendations = self.createComponentLayoutRecommendations(services, hosts)
recommendations = {
"Versions": {"stack_name": stackName, "stack_version": stackVersion},
@@ -320,7 +327,7 @@ class DefaultStackAdvisor(StackAdvisor):
return recommendations
- def provideLayout(self, services, hosts):
+ def createComponentLayoutRecommendations(self, services, hosts):
recommendations = {
"blueprint": {
@@ -340,19 +347,19 @@ class DefaultStackAdvisor(StackAdvisor):
#extend 'hostsComponentsMap' with MASTER components
for service in services["services"]:
- masterComponents = [component for component in service["components"] if self.isMaster(component)]
+ masterComponents = [component for component in service["components"] if self.isMasterComponent(component)]
for component in masterComponents:
componentName = component["StackServiceComponents"]["component_name"]
- if self.isAlreadyPopulated(component):
+ if self.isComponentHostsPopulated(component):
hostsForComponent = component["StackServiceComponents"]["hostnames"]
else:
availableHosts = hostsList
- if len(hostsList) > 1 and self.isNotPreferableOnAmbariServerHost(component):
+ if len(hostsList) > 1 and self.isComponentNotPreferableOnAmbariServerHost(component):
availableHosts = [hostName for hostName in hostsList if not self.isLocalHost(hostName)]
- if self.isMasterWithMultipleInstances(component):
- hostsCount = self.defaultNoOfMasterHosts(component)
+ if self.isMasterComponentWithMultipleInstances(component):
+ hostsCount = self.getMinComponentCount(component)
if hostsCount > 1: # get first 'hostsCount' available hosts
if len(availableHosts) < hostsCount:
hostsCount = len(availableHosts)
@@ -369,17 +376,17 @@ class DefaultStackAdvisor(StackAdvisor):
#extend 'hostsComponentsMap' with Slave and Client Components
componentsListList = [service["components"] for service in services["services"]]
componentsList = [item for sublist in componentsListList for item in sublist]
- usedHostsListList = [component["StackServiceComponents"]["hostnames"] for component in componentsList if not self.isNotValuable(component)]
+ usedHostsListList = [component["StackServiceComponents"]["hostnames"] for component in componentsList if not self.isComponentNotValuable(component)]
utilizedHosts = [item for sublist in usedHostsListList for item in sublist]
freeHosts = [hostName for hostName in hostsList if hostName not in utilizedHosts]
for service in services["services"]:
slaveClientComponents = [component for component in service["components"]
- if self.isSlave(component) or self.isClient(component)]
+ if self.isSlaveComponent(component) or self.isClientComponent(component)]
for component in slaveClientComponents:
componentName = component["StackServiceComponents"]["component_name"]
- if self.isAlreadyPopulated(component):
+ if self.isComponentHostsPopulated(component):
hostsForComponent = component["StackServiceComponents"]["hostnames"]
elif component["StackServiceComponents"]["cardinality"] == "ALL":
hostsForComponent = hostsList
@@ -388,7 +395,7 @@ class DefaultStackAdvisor(StackAdvisor):
hostsForComponent = hostsList[-1:]
else: # len(freeHosts) >= 1
hostsForComponent = freeHosts
- if self.isClient(component):
+ if self.isClientComponent(component):
hostsForComponent = freeHosts[0:1]
#extend 'hostsComponentsMap' with 'hostsForComponent'
@@ -410,7 +417,7 @@ class DefaultStackAdvisor(StackAdvisor):
return recommendations
pass
- def prepareValidationResponse(self, services, validationItems):
+ def createValidationResponse(self, services, validationItems):
"""Returns array of Validation objects about issues with hostnames components assigned to"""
stackName = services["Versions"]["stack_name"]
stackVersion = services["Versions"]["stack_version"]
@@ -424,18 +431,18 @@ class DefaultStackAdvisor(StackAdvisor):
def validateComponentLayout(self, services, hosts):
"""Returns array of Validation objects about issues with hostnames components assigned to"""
- validationItems = self.getLayoutValidationItems(services, hosts)
- return self.prepareValidationResponse(services, validationItems)
+ validationItems = self.getComponentLayoutValidations(services, hosts)
+ return self.createValidationResponse(services, validationItems)
def validateConfigurations(self, services, hosts):
"""Returns array of Validation objects about issues with hostnames components assigned to"""
validationItems = self.getConfigurationsValidationItems(services, hosts)
- return self.prepareValidationResponse(services, validationItems)
+ return self.createValidationResponse(services, validationItems)
- def getLayoutValidationItems(self, services, hosts):
+ def getComponentLayoutValidations(self, services, hosts):
return []
- def getClusterData(self, servicesList, hosts, components):
+ def getConfigurationClusterSummary(self, servicesList, hosts, components):
pass
def getConfigurationsValidationItems(self, services, hosts):
@@ -450,7 +457,7 @@ class DefaultStackAdvisor(StackAdvisor):
for service in services["services"]
for component in service["components"]]
- clusterData = self.getClusterData(servicesList, hosts, components)
+ clusterSummary = self.getConfigurationClusterSummary(servicesList, hosts, components)
recommendations = {
"Versions": {"stack_name": stackName, "stack_version": stackVersion},
@@ -470,32 +477,32 @@ class DefaultStackAdvisor(StackAdvisor):
configurations = recommendations["recommendations"]["blueprint"]["configurations"]
for service in servicesList:
- calculation = self.recommendServiceConfigurations(service)
+ calculation = self.getServiceConfigurationRecommender(service)
if calculation is not None:
- calculation(configurations, clusterData)
+ calculation(configurations, clusterSummary)
return recommendations
- def recommendServiceConfigurations(self, service):
- return self.getServiceConfiguratorDict().get(service, None)
+ def getServiceConfigurationRecommender(self, service):
+ return self.getServiceConfigurationRecommenderDict().get(service, None)
- def getServiceConfiguratorDict(self):
+ def getServiceConfigurationRecommenderDict(self):
return {}
# Recommendation helper methods
- def isAlreadyPopulated(self, component):
+ def isComponentHostsPopulated(self, component):
hostnames = self.getComponentAttribute(component, "hostnames")
if hostnames is not None:
return len(hostnames) > 0
return False
- def isClient(self, component):
+ def isClientComponent(self, component):
return self.getComponentAttribute(component, "component_category") == 'CLIENT'
- def isSlave(self, component):
+ def isSlaveComponent(self, component):
return self.getComponentAttribute(component, "component_category") == 'SLAVE'
- def isMaster(self, component):
+ def isMasterComponent(self, component):
return self.getComponentAttribute(component, "is_master")
def getComponentAttribute(self, component, attribute):
@@ -507,22 +514,22 @@ class DefaultStackAdvisor(StackAdvisor):
def isLocalHost(self, hostName):
return socket.getfqdn(hostName) == socket.getfqdn()
- def isMasterWithMultipleInstances(self, component):
+ def isMasterComponentWithMultipleInstances(self, component):
componentName = self.getComponentName(component)
masters = self.getMastersWithMultipleInstances()
return componentName in masters
- def isNotValuable(self, component):
+ def isComponentNotValuable(self, component):
componentName = self.getComponentName(component)
service = self.getNotValuableComponents()
return componentName in service
- def defaultNoOfMasterHosts(self, component):
+ def getMinComponentCount(self, component):
componentName = self.getComponentName(component)
- return self.cardinality(componentName)["min"]
+ return self.getComponentCardinality(componentName)["min"]
# Helper dictionaries
- def cardinality(self, componentName):
+ def getComponentCardinality(self, componentName):
return self.getCardinalitiesDict().get(componentName, {"min": 1, "max": 1})
def getHostForComponent(self, component, hostsList, hostsComponentsMap):
@@ -531,7 +538,7 @@ class DefaultStackAdvisor(StackAdvisor):
if len(hostsList) == 1:
return hostsList[0]
else:
- scheme = self.defineSelectionScheme(componentName)
+ scheme = self.getComponentLayoutScheme(componentName)
if scheme is not None:
for key in scheme.keys():
if isinstance(key, ( int, long )):
@@ -543,13 +550,16 @@ class DefaultStackAdvisor(StackAdvisor):
hostOccupations = dict((host, len(components)) for host, components in hostComponentsMap.iteritems() if host in hostsList)
return min(hostOccupations, key=hostOccupations.get)
- def defineSelectionScheme(self, componentName):
- return self.selectionSchemes().get(componentName, None)
+ def getComponentLayoutScheme(self, componentName):
+ """
+ Provides a scheme for laying out given component on different number of hosts.
+ """
+ return self.getComponentLayoutSchemes().get(componentName, None)
def getComponentName(self, component):
return self.getComponentAttribute(component, "component_name")
- def isNotPreferableOnAmbariServerHost(self, component):
+ def isComponentNotPreferableOnAmbariServerHost(self, component):
componentName = self.getComponentName(component)
service = self.getNotPreferableOnServerComponents()
return componentName in service
@@ -566,5 +576,11 @@ class DefaultStackAdvisor(StackAdvisor):
def getCardinalitiesDict(self):
return {}
- def selectionSchemes(self):
+ def getComponentLayoutSchemes(self):
+ """
+ Provides layout scheme dictionaries for components.
+
+ The scheme dictionary basically maps the number of hosts to
+ host index where component should exist.
+ """
return {}