You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sr...@apache.org on 2014/09/04 01:16:37 UTC
git commit: AMBARI-7068. BE: HDP 2.0.6 stack_advisor.py should be
tolerant of services not owned by it
Repository: ambari
Updated Branches:
refs/heads/trunk 24dea68d6 -> dc269bb15
AMBARI-7068. BE: HDP 2.0.6 stack_advisor.py should be tolerant of services not owned by it
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/dc269bb1
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/dc269bb1
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/dc269bb1
Branch: refs/heads/trunk
Commit: dc269bb1588bee0bf6619f8a70ad1f144591ca09
Parents: 24dea68
Author: Srimanth Gunturi <sg...@hortonworks.com>
Authored: Wed Sep 3 14:28:25 2014 -0700
Committer: Srimanth Gunturi <sg...@hortonworks.com>
Committed: Wed Sep 3 16:16:20 2014 -0700
----------------------------------------------------------------------
.../src/main/resources/scripts/stack_advisor.py | 19 +-
.../stacks/HDP/1.3.2/services/stack_advisor.py | 377 +++++--------------
.../stacks/HDP/1.3.3/services/stack_advisor.py | 4 -
.../stacks/HDP/1.3/services/stack_advisor.py | 4 -
.../stacks/HDP/2.0.6/services/stack_advisor.py | 308 +++------------
.../stacks/HDP/2.1/services/stack_advisor.py | 64 ++--
.../src/main/resources/stacks/stack_advisor.py | 271 +++++++++++++
.../src/test/python/TestStackAdvisor.py | 323 ++++++++++++++--
.../stacks/2.0.6/common/test_stack_advisor.py | 2 +-
9 files changed, 738 insertions(+), 634 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/dc269bb1/ambari-server/src/main/resources/scripts/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/scripts/stack_advisor.py b/ambari-server/src/main/resources/scripts/stack_advisor.py
index ed50866..566fe7f 100755
--- a/ambari-server/src/main/resources/scripts/stack_advisor.py
+++ b/ambari-server/src/main/resources/scripts/stack_advisor.py
@@ -34,6 +34,7 @@ USAGE = "Usage: <action> <hosts_file> <services_file>\nPossible actions are: {0}
SCRIPT_DIRECTORY = os.path.dirname(os.path.abspath(__file__))
STACK_ADVISOR_PATH_TEMPLATE = os.path.join(SCRIPT_DIRECTORY, '../stacks/stack_advisor.py')
+STACK_ADVISOR_DEFAULT_IMPL_CLASS = 'DefaultStackAdvisor'
STACK_ADVISOR_IMPL_PATH_TEMPLATE = os.path.join(SCRIPT_DIRECTORY, './../stacks/{0}/{1}/services/stack_advisor.py')
STACK_ADVISOR_IMPL_CLASS_TEMPLATE = '{0}{1}StackAdvisor'
@@ -113,9 +114,10 @@ def instantiateStackAdvisor(stackName, stackVersion, parentVersions):
"""Instantiates StackAdvisor implementation for the specified Stack"""
import imp
- stackAdvisorPath = STACK_ADVISOR_PATH_TEMPLATE.format(stackName)
- with open(stackAdvisorPath, 'rb') as fp:
- stack_advisor = imp.load_module( 'stack_advisor', fp, stackAdvisorPath, ('.py', 'rb', imp.PY_SOURCE) )
+ with open(STACK_ADVISOR_PATH_TEMPLATE, 'rb') as fp:
+ default_stack_advisor = imp.load_module('stack_advisor', fp, STACK_ADVISOR_PATH_TEMPLATE, ('.py', 'rb', imp.PY_SOURCE))
+ className = STACK_ADVISOR_DEFAULT_IMPL_CLASS
+ stack_advisor = default_stack_advisor
versions = [stackVersion]
versions.extend(parentVersions)
@@ -123,20 +125,21 @@ def instantiateStackAdvisor(stackName, stackVersion, parentVersions):
for version in reversed(versions):
try:
path = STACK_ADVISOR_IMPL_PATH_TEMPLATE.format(stackName, version)
- className = STACK_ADVISOR_IMPL_CLASS_TEMPLATE.format(stackName, version.replace('.', ''))
with open(path, 'rb') as fp:
- stack_advisor_impl = imp.load_module('stack_advisor_impl', fp, path, ('.py', 'rb', imp.PY_SOURCE))
+ stack_advisor = imp.load_module('stack_advisor_impl', fp, path, ('.py', 'rb', imp.PY_SOURCE))
+ className = STACK_ADVISOR_IMPL_CLASS_TEMPLATE.format(stackName, version.replace('.', ''))
print "StackAdvisor implementation for stack {0}, version {1} was loaded".format(stackName, version)
- except Exception, e:
+ except Exception:
print "StackAdvisor implementation for stack {0}, version {1} was not found".format(stackName, version)
try:
- clazz = getattr(stack_advisor_impl, className)
+ clazz = getattr(stack_advisor, className)
+ print "Returning " + className + " implementation"
return clazz()
except Exception, e:
print "Returning default implementation"
- return stack_advisor.StackAdvisor()
+ return default_stack_advisor.DefaultStackAdvisor()
if __name__ == '__main__':
http://git-wip-us.apache.org/repos/asf/ambari/blob/dc269bb1/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/stack_advisor.py
index 17a26e2..c9b119f 100644
--- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/stack_advisor.py
@@ -18,169 +18,19 @@ limitations under the License.
"""
import re
-import socket
import sys
-from stack_advisor import StackAdvisor
+from stack_advisor import DefaultStackAdvisor
-class HDP132StackAdvisor(StackAdvisor):
+class HDP132StackAdvisor(DefaultStackAdvisor):
- def recommendComponentLayout(self, services, hosts):
- """
- Returns Services object with hostnames array populated for components
- If hostnames are populated for some components (partial blueprint) - these components will not be processed
- """
- stackName = services["Versions"]["stack_name"]
- stackVersion = services["Versions"]["stack_version"]
- hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
- servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
-
- recommendations = {
- "Versions": {"stack_name": stackName, "stack_version": stackVersion},
- "hosts": hostsList,
- "services": servicesList,
- "recommendations": {
- "blueprint": {
- "host_groups": [ ]
- },
- "blueprint_cluster_binding": {
- "host_groups": [ ]
- }
- }
- }
-
- hostsComponentsMap = {}
-
- #extend 'hostsComponentsMap' with MASTER components
- for service in services["services"]:
- masterComponents = [component for component in service["components"] if isMaster(component)]
- for component in masterComponents:
- componentName = component["StackServiceComponents"]["component_name"]
- hostsForComponent = []
-
- if isAlreadyPopulated(component):
- hostsForComponent = component["StackServiceComponents"]["hostnames"]
- else:
- availableHosts = hostsList
- if len(hostsList) > 1 and self.isNotPreferableOnAmbariServerHost(component):
- availableHosts = [hostName for hostName in hostsList if not isLocalHost(hostName)]
-
- if isMasterWithMultipleInstances(component):
- hostsCount = defaultNoOfMasterHosts(component)
- if hostsCount > 1: # get first 'hostsCount' available hosts
- if len(availableHosts) < hostsCount:
- hostsCount = len(availableHosts)
- hostsForComponent = availableHosts[:hostsCount]
- else:
- hostsForComponent = [self.getHostForComponent(component, availableHosts)]
- else:
- hostsForComponent = [self.getHostForComponent(component, availableHosts)]
-
- #extend 'hostsComponentsMap' with 'hostsForComponent'
- for hostName in hostsForComponent:
- if hostName not in hostsComponentsMap:
- hostsComponentsMap[hostName] = []
- hostsComponentsMap[hostName].append( { "name":componentName } )
-
- #extend 'hostsComponentsMap' with Slave and Client Components
- componentsListList = [service["components"] for service in services["services"]]
- componentsList = [item for sublist in componentsListList for item in sublist]
- usedHostsListList = [component["StackServiceComponents"]["hostnames"] for component in componentsList if not self.isNotValuable(component)]
- utilizedHosts = [item for sublist in usedHostsListList for item in sublist]
- freeHosts = [hostName for hostName in hostsList if hostName not in utilizedHosts]
-
- for service in services["services"]:
- slaveClientComponents = [component for component in service["components"] if isSlave(component) or isClient(component)]
- for component in slaveClientComponents:
- componentName = component["StackServiceComponents"]["component_name"]
- hostsForComponent = []
-
- if isAlreadyPopulated(component):
- hostsForComponent = component["StackServiceComponents"]["hostnames"]
- elif component["StackServiceComponents"]["cardinality"] == "ALL":
- hostsForComponent = hostsList
- else:
- if len(freeHosts) == 0:
- hostsForComponent = hostsList[-1:]
- else: # len(freeHosts) >= 1
- hostsForComponent = freeHosts
- if isClient(component):
- hostsForComponent = freeHosts[0:1]
-
- #extend 'hostsComponentsMap' with 'hostsForComponent'
- for hostName in hostsForComponent:
- if hostName not in hostsComponentsMap:
- hostsComponentsMap[hostName] = []
- hostsComponentsMap[hostName].append( { "name": componentName } )
-
- #prepare 'host-group's from 'hostsComponentsMap'
- host_groups = recommendations["recommendations"]["blueprint"]["host_groups"]
- bindings = recommendations["recommendations"]["blueprint_cluster_binding"]["host_groups"]
- index = 0
- for key in hostsComponentsMap.keys():
- index += 1
- host_group_name = "host-group-{0}".format(index)
- host_groups.append( { "name": host_group_name, "components": hostsComponentsMap[key] } )
- bindings.append( { "name": host_group_name, "hosts": [{ "fqdn": socket.getfqdn(key) }] } )
-
- return recommendations
- pass
-
- def getHostForComponent(self, component, hostsList):
- componentName = component["StackServiceComponents"]["component_name"]
- scheme = self.defineSelectionScheme(componentName)
-
- if len(hostsList) == 1:
- return hostsList[0]
- else:
- for key in scheme.keys():
- if isinstance(key, ( int, long )):
- if len(hostsList) < key:
- return hostsList[scheme[key]]
- return hostsList[scheme['else']]
-
- def defineSelectionScheme(self, componentName):
- scheme = self.selectionScheme(componentName)
- if scheme is None:
- scheme = {"else": 0}
- return scheme
-
- def selectionScheme(self, componentName):
- return {
- 'NAMENODE': {"else": 0},
- 'SECONDARY_NAMENODE': {"else": 1},
- 'HBASE_MASTER': {6: 0, 31: 2, "else": 3},
-
- 'HISTORYSERVER': {31: 1, "else": 2},
- 'RESOURCEMANAGER': {31: 1, "else": 2},
-
- 'OOZIE_SERVER': {6: 1, 31: 2, "else": 3},
-
- 'HIVE_SERVER': {6: 1, 31: 2, "else": 4},
- 'HIVE_METASTORE': {6: 1, 31: 2, "else": 4},
- 'WEBHCAT_SERVER': {6: 1, 31: 2, "else": 4},
- }.get(componentName, None)
-
- def isNotPreferableOnAmbariServerHost(self, component):
- componentName = component["StackServiceComponents"]["component_name"]
- service = ['GANGLIA_SERVER', 'NAGIOS_SERVER']
- return componentName in service
-
- def validateComponentLayout(self, services, hosts):
+ def getLayoutValidationItems(self, services, hosts):
"""Returns array of Validation objects about issues with hostnames components assigned to"""
- stackName = services["Versions"]["stack_name"]
- stackVersion = services["Versions"]["stack_version"]
-
- validations = {
- "Versions": {"stack_name": stackName, "stack_version": stackVersion},
- "items": [ ]
- }
- items = validations["items"]
+ items = []
# Validating NAMENODE and SECONDARY_NAMENODE are on different hosts if possible
hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
hostsCount = len(hostsList)
- servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
componentsListList = [service["components"] for service in services["services"]]
componentsList = [item for sublist in componentsListList for item in sublist]
@@ -198,32 +48,28 @@ class HDP132StackAdvisor(StackAdvisor):
# Validating cardinality
for component in componentsList:
if component["StackServiceComponents"]["cardinality"] is not None:
- componentName = component["StackServiceComponents"]["component_name"]
- componentHostsCount = 0
- if component["StackServiceComponents"]["hostnames"] is not None:
- componentHostsCount = len(component["StackServiceComponents"]["hostnames"])
- cardinality = str(component["StackServiceComponents"]["cardinality"])
- # cardinality types: null, 1+, 1-2, 1, ALL
- hostsMax = -sys.maxint - 1
- hostsMin = sys.maxint
- hostsMin = 0
- hostsMax = 0
- if "+" in cardinality:
- hostsMin = int(cardinality[:-1])
- hostsMax = sys.maxint
- elif "-" in cardinality:
- nums = cardinality.split("-")
- hostsMin = int(nums[0])
- hostsMax = int(nums[1])
- elif "ALL" == cardinality:
- hostsMin = hostsCount
- hostsMax = hostsCount
- else:
- hostsMin = int(cardinality)
- hostsMax = int(cardinality)
-
- if componentHostsCount > hostsMax or componentHostsCount < hostsMin:
- items.append( { "type": 'host-component', "level": 'ERROR', "message": 'Cardinality violation, cardinality={0}, hosts count={1}'.format(cardinality, str(componentHostsCount)), "component-name": str(componentName) } )
+ componentName = component["StackServiceComponents"]["component_name"]
+ componentHostsCount = 0
+ if component["StackServiceComponents"]["hostnames"] is not None:
+ componentHostsCount = len(component["StackServiceComponents"]["hostnames"])
+ cardinality = str(component["StackServiceComponents"]["cardinality"])
+ # cardinality types: null, 1+, 1-2, 1, ALL
+ if "+" in cardinality:
+ hostsMin = int(cardinality[:-1])
+ hostsMax = sys.maxint
+ elif "-" in cardinality:
+ nums = cardinality.split("-")
+ hostsMin = int(nums[0])
+ hostsMax = int(nums[1])
+ elif "ALL" == cardinality:
+ hostsMin = hostsCount
+ hostsMax = hostsCount
+ else:
+ hostsMin = int(cardinality)
+ hostsMax = int(cardinality)
+
+ if componentHostsCount > hostsMax or componentHostsCount < hostsMin:
+ items.append( { "type": 'host-component', "level": 'ERROR', "message": 'Cardinality violation, cardinality={0}, hosts count={1}'.format(cardinality, str(componentHostsCount)), "component-name": str(componentName) } )
# Validating host-usage
usedHostsListList = [component["StackServiceComponents"]["hostnames"] for component in componentsList if not self.isNotValuable(component)]
@@ -232,52 +78,11 @@ class HDP132StackAdvisor(StackAdvisor):
for host in nonUsedHostsList:
items.append( { "type": 'host-component', "level": 'ERROR', "message": 'Host is not used', "host": str(host) } )
- return validations
- pass
-
- def isNotValuable(self, component):
- componentName = component["StackServiceComponents"]["component_name"]
- service = ['JOURNALNODE', 'ZKFC', 'GANGLIA_MONITOR']
- return componentName in service
-
- def recommendConfigurations(self, services, hosts):
- stackName = services["Versions"]["stack_name"]
- stackVersion = services["Versions"]["stack_version"]
- hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
- servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
- components = [component["StackServiceComponents"]["component_name"]
- for service in services["services"]
- for component in service["components"]]
-
- clusterData = self.getClusterData(servicesList, hosts, components)
-
- recommendations = {
- "Versions": {"stack_name": stackName, "stack_version": stackVersion},
- "hosts": hostsList,
- "services": servicesList,
- "recommendations": {
- "blueprint": {
- "configurations": {},
- "host_groups": []
- },
- "blueprint_cluster_binding": {
- "host_groups": []
- }
- }
- }
+ return items
- configurations = recommendations["recommendations"]["blueprint"]["configurations"]
-
- for service in servicesList:
- calculation = self.recommendServiceConfigurations(service)
- if calculation is not None:
- calculation(configurations, clusterData)
-
- return recommendations
-
- def recommendServiceConfigurations(self, service):
+ def getServiceConfiguratorDict(self):
return {
- }.get(service, None)
+ }
def putProperty(self, config, configType):
config[configType] = {"properties": {}}
@@ -361,17 +166,9 @@ class HDP132StackAdvisor(StackAdvisor):
return cluster
-
- def validateConfigurations(self, services, hosts):
+ def getConfigurationsValidationItems(self, services, hosts):
"""Returns array of Validation objects about issues with configuration values provided in services"""
- stackName = services["Versions"]["stack_name"]
- stackVersion = services["Versions"]["stack_version"]
-
- validations = {
- "Versions": {"stack_name": stackName, "stack_version": stackVersion},
- "items": [ ]
- }
- items = validations["items"]
+ items = []
recommendations = self.recommendConfigurations(services, hosts)
recommendedDefaults = recommendations["recommendations"]["blueprint"]["configurations"]
@@ -388,50 +185,88 @@ class HDP132StackAdvisor(StackAdvisor):
if siteProperties is not None:
resultItems = method(siteProperties, recommendedDefaults[siteName]["properties"])
items.extend(resultItems)
- return validations
- pass
+ return items
+
+ def getServiceConfigurationValidators(self):
+ return {}
def validateServiceConfigurations(self, serviceName):
- return {
- }.get(serviceName, None)
+ return self.getServiceConfigurationValidators().get(serviceName, None)
- def toConfigurationValidationErrors(self, items, siteName):
+ def toConfigurationValidationProblems(self, validationProblems, siteName):
result = []
- for item in items:
- if item["message"] is not None:
- error = { "type": 'configuration', "level": 'ERROR', "message": item["message"], "config-type": siteName, "config-name": item["config-name"] }
- result.append(error)
+ for validationProblem in validationProblems:
+ validationItem = validationProblem.get("item", None)
+ if validationItem is not None:
+ problem = {"type": 'configuration', "level": validationItem["level"], "message": validationItem["message"],
+ "config-type": siteName, "config-name": validationProblem["config-name"] }
+ result.append(problem)
return result
+ def getWarnItem(self, message):
+ return {"level": "WARN", "message": message}
+
+ def getErrorItem(self, message):
+ return {"level": "ERROR", "message": message}
+
def validatorLessThenDefaultValue(self, properties, recommendedDefaults, propertyName):
if not propertyName in properties:
- return "Value should be set"
+ return self.getErrorItem("Value should be set")
value = to_number(properties[propertyName])
if value is None:
- return "Value should be integer"
+ return self.getErrorItem("Value should be integer")
defaultValue = to_number(recommendedDefaults[propertyName])
if defaultValue is None:
return None
if value < defaultValue:
- return "Value is less than the recommended default of {0}".format(defaultValue)
+ return self.getWarnItem("Value is less than the recommended default of {0}".format(defaultValue))
return None
def validateXmxValue(self, properties, recommendedDefaults, propertyName):
if not propertyName in properties:
- return "Value should be set"
+ return self.getErrorItem("Value should be set")
value = properties[propertyName]
defaultValue = recommendedDefaults[propertyName]
if defaultValue is None:
- return "Config's default value can't be null or undefined"
+ return self.getErrorItem("Config's default value can't be null or undefined")
if not checkXmxValueFormat(value):
- return 'Invalid value format'
+ return self.getErrorItem('Invalid value format')
valueInt = formatXmxSizeToBytes(getXmxSize(value))
defaultValueXmx = getXmxSize(defaultValue)
defaultValueInt = formatXmxSizeToBytes(defaultValueXmx)
if valueInt < defaultValueInt:
- return "Value is less than the recommended default of -Xmx" + defaultValueXmx
+ return self.getWarnItem("Value is less than the recommended default of -Xmx" + defaultValueXmx)
return None
+ def getMastersWithMultipleInstances(self):
+ return ['ZOOKEEPER_SERVER', 'HBASE_MASTER']
+ def getNotValuableComponents(self):
+ return ['JOURNALNODE', 'ZKFC', 'GANGLIA_MONITOR']
+
+ def getNotPreferableOnServerComponents(self):
+ return ['GANGLIA_SERVER', 'NAGIOS_SERVER']
+
+ def getCardinalitiesDict(self):
+ return {
+ 'ZOOKEEPER_SERVER': {"min": 3},
+ 'HBASE_MASTER': {"min": 1},
+ }
+
+ def selectionSchemes(self):
+ return {
+ 'NAMENODE': {"else": 0},
+ 'SECONDARY_NAMENODE': {"else": 1},
+ 'HBASE_MASTER': {6: 0, 31: 2, "else": 3},
+
+ 'HISTORYSERVER': {31: 1, "else": 2},
+ 'RESOURCEMANAGER': {31: 1, "else": 2},
+
+ 'OOZIE_SERVER': {6: 1, 31: 2, "else": 3},
+
+ 'HIVE_SERVER': {6: 1, 31: 2, "else": 4},
+ 'HIVE_METASTORE': {6: 1, 31: 2, "else": 4},
+ 'WEBHCAT_SERVER': {6: 1, 31: 2, "else": 4},
+ }
# Validation helper methods
def getSiteProperties(configurations, siteName):
@@ -474,46 +309,6 @@ def formatXmxSizeToBytes(value):
modifier == 'g': 1024 * 1024 * 1024,
modifier == 't': 1024 * 1024 * 1024 * 1024,
modifier == 'p': 1024 * 1024 * 1024 * 1024 * 1024
- }[1]
+ }[1]
return to_number(value) * m
-
-# Recommendation helper methods
-def isAlreadyPopulated(component):
- if component["StackServiceComponents"]["hostnames"] is not None:
- return len(component["StackServiceComponents"]["hostnames"]) > 0
- return False
-
-def isClient(component):
- return component["StackServiceComponents"]["component_category"] == 'CLIENT'
-
-def isSlave(component):
- componentName = component["StackServiceComponents"]["component_name"]
- isSlave = component["StackServiceComponents"]["component_category"] == 'SLAVE'
- return isSlave
-
-def isMaster(component):
- componentName = component["StackServiceComponents"]["component_name"]
- isMaster = component["StackServiceComponents"]["is_master"]
- return isMaster
-
-def isLocalHost(hostName):
- return socket.getfqdn(hostName) == socket.getfqdn()
-
-def isMasterWithMultipleInstances(component):
- componentName = component["StackServiceComponents"]["component_name"]
- masters = ['ZOOKEEPER_SERVER', 'HBASE_MASTER']
- return componentName in masters
-
-def defaultNoOfMasterHosts(component):
- componentName = component["StackServiceComponents"]["component_name"]
- return cardinality(componentName)[min]
-
-
-# Helper dictionaries
-def cardinality(componentName):
- return {
- 'ZOOKEEPER_SERVER': {min: 3},
- 'HBASE_MASTER': {min: 1},
- }.get(componentName, {min:1, max:1})
-
http://git-wip-us.apache.org/repos/asf/ambari/blob/dc269bb1/ambari-server/src/main/resources/stacks/HDP/1.3.3/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/1.3.3/services/stack_advisor.py
index f91efd8..40fac9c 100644
--- a/ambari-server/src/main/resources/stacks/HDP/1.3.3/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/1.3.3/services/stack_advisor.py
@@ -17,9 +17,5 @@ See the License for the specific language governing permissions and
limitations under the License.
"""
-import socket
-
-from stack_advisor import StackAdvisor
-
class HDP133StackAdvisor(HDP132StackAdvisor):
pass
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/dc269bb1/ambari-server/src/main/resources/stacks/HDP/1.3/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/1.3/services/stack_advisor.py
index 998ecaa..9360646 100644
--- a/ambari-server/src/main/resources/stacks/HDP/1.3/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/1.3/services/stack_advisor.py
@@ -17,9 +17,5 @@ See the License for the specific language governing permissions and
limitations under the License.
"""
-import socket
-
-from stack_advisor import StackAdvisor
-
class HDP13StackAdvisor(HDP133StackAdvisor):
pass
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/dc269bb1/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index 4eb8f02..dd4fae3 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -18,169 +18,19 @@ limitations under the License.
"""
import re
-import socket
import sys
-from stack_advisor import StackAdvisor
+from stack_advisor import DefaultStackAdvisor
-class HDP206StackAdvisor(StackAdvisor):
+class HDP206StackAdvisor(DefaultStackAdvisor):
- def recommendComponentLayout(self, services, hosts):
- """
- Returns Services object with hostnames array populated for components
- If hostnames are populated for some components (partial blueprint) - these components will not be processed
- """
- stackName = services["Versions"]["stack_name"]
- stackVersion = services["Versions"]["stack_version"]
- hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
- servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
-
- recommendations = {
- "Versions": {"stack_name": stackName, "stack_version": stackVersion},
- "hosts": hostsList,
- "services": servicesList,
- "recommendations": {
- "blueprint": {
- "host_groups": [ ]
- },
- "blueprint_cluster_binding": {
- "host_groups": [ ]
- }
- }
- }
-
- hostsComponentsMap = {}
-
- #extend 'hostsComponentsMap' with MASTER components
- for service in services["services"]:
- masterComponents = [component for component in service["components"] if isMaster(component)]
- for component in masterComponents:
- componentName = component["StackServiceComponents"]["component_name"]
- hostsForComponent = []
-
- if isAlreadyPopulated(component):
- hostsForComponent = component["StackServiceComponents"]["hostnames"]
- else:
- availableHosts = hostsList
- if len(hostsList) > 1 and self.isNotPreferableOnAmbariServerHost(component):
- availableHosts = [hostName for hostName in hostsList if not isLocalHost(hostName)]
-
- if isMasterWithMultipleInstances(component):
- hostsCount = defaultNoOfMasterHosts(component)
- if hostsCount > 1: # get first 'hostsCount' available hosts
- if len(availableHosts) < hostsCount:
- hostsCount = len(availableHosts)
- hostsForComponent = availableHosts[:hostsCount]
- else:
- hostsForComponent = [self.getHostForComponent(component, availableHosts)]
- else:
- hostsForComponent = [self.getHostForComponent(component, availableHosts)]
-
- #extend 'hostsComponentsMap' with 'hostsForComponent'
- for hostName in hostsForComponent:
- if hostName not in hostsComponentsMap:
- hostsComponentsMap[hostName] = []
- hostsComponentsMap[hostName].append( { "name":componentName } )
-
- #extend 'hostsComponentsMap' with Slave and Client Components
- componentsListList = [service["components"] for service in services["services"]]
- componentsList = [item for sublist in componentsListList for item in sublist]
- usedHostsListList = [component["StackServiceComponents"]["hostnames"] for component in componentsList if not self.isNotValuable(component)]
- utilizedHosts = [item for sublist in usedHostsListList for item in sublist]
- freeHosts = [hostName for hostName in hostsList if hostName not in utilizedHosts]
-
- for service in services["services"]:
- slaveClientComponents = [component for component in service["components"] if isSlave(component) or isClient(component)]
- for component in slaveClientComponents:
- componentName = component["StackServiceComponents"]["component_name"]
- hostsForComponent = []
-
- if isAlreadyPopulated(component):
- hostsForComponent = component["StackServiceComponents"]["hostnames"]
- elif component["StackServiceComponents"]["cardinality"] == "ALL":
- hostsForComponent = hostsList
- else:
- if len(freeHosts) == 0:
- hostsForComponent = hostsList[-1:]
- else: # len(freeHosts) >= 1
- hostsForComponent = freeHosts
- if isClient(component):
- hostsForComponent = freeHosts[0:1]
-
- #extend 'hostsComponentsMap' with 'hostsForComponent'
- for hostName in hostsForComponent:
- if hostName not in hostsComponentsMap:
- hostsComponentsMap[hostName] = []
- hostsComponentsMap[hostName].append( { "name": componentName } )
-
- #prepare 'host-group's from 'hostsComponentsMap'
- host_groups = recommendations["recommendations"]["blueprint"]["host_groups"]
- bindings = recommendations["recommendations"]["blueprint_cluster_binding"]["host_groups"]
- index = 0
- for key in hostsComponentsMap.keys():
- index += 1
- host_group_name = "host-group-{0}".format(index)
- host_groups.append( { "name": host_group_name, "components": hostsComponentsMap[key] } )
- bindings.append( { "name": host_group_name, "hosts": [{ "fqdn": socket.getfqdn(key) }] } )
-
- return recommendations
- pass
-
- def getHostForComponent(self, component, hostsList):
- componentName = component["StackServiceComponents"]["component_name"]
- scheme = self.defineSelectionScheme(componentName)
-
- if len(hostsList) == 1:
- return hostsList[0]
- else:
- for key in scheme.keys():
- if isinstance(key, ( int, long )):
- if len(hostsList) < key:
- return hostsList[scheme[key]]
- return hostsList[scheme['else']]
-
- def defineSelectionScheme(self, componentName):
- scheme = self.selectionScheme(componentName)
- if scheme is None:
- scheme = {"else": 0}
- return scheme
-
- def selectionScheme(self, componentName):
- return {
- 'NAMENODE': {"else": 0},
- 'SECONDARY_NAMENODE': {"else": 1},
- 'HBASE_MASTER': {6: 0, 31: 2, "else": 3},
-
- 'HISTORYSERVER': {31: 1, "else": 2},
- 'RESOURCEMANAGER': {31: 1, "else": 2},
-
- 'OOZIE_SERVER': {6: 1, 31: 2, "else": 3},
-
- 'HIVE_SERVER': {6: 1, 31: 2, "else": 4},
- 'HIVE_METASTORE': {6: 1, 31: 2, "else": 4},
- 'WEBHCAT_SERVER': {6: 1, 31: 2, "else": 4},
- }.get(componentName, None)
-
- def isNotPreferableOnAmbariServerHost(self, component):
- componentName = component["StackServiceComponents"]["component_name"]
- service = ['GANGLIA_SERVER', 'NAGIOS_SERVER']
- return componentName in service
-
- def validateComponentLayout(self, services, hosts):
+ def getLayoutValidationItems(self, services, hosts):
"""Returns array of Validation objects about issues with hostnames components assigned to"""
- stackName = services["Versions"]["stack_name"]
- stackVersion = services["Versions"]["stack_version"]
-
- validations = {
- "Versions": {"stack_name": stackName, "stack_version": stackVersion},
- "items": [ ]
- }
- items = validations["items"]
+ items = []
# Validating NAMENODE and SECONDARY_NAMENODE are on different hosts if possible
hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
hostsCount = len(hostsList)
- servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
componentsListList = [service["components"] for service in services["services"]]
componentsList = [item for sublist in componentsListList for item in sublist]
@@ -204,10 +54,6 @@ class HDP206StackAdvisor(StackAdvisor):
componentHostsCount = len(component["StackServiceComponents"]["hostnames"])
cardinality = str(component["StackServiceComponents"]["cardinality"])
# cardinality types: null, 1+, 1-2, 1, ALL
- hostsMax = -sys.maxint - 1
- hostsMin = sys.maxint
- hostsMin = 0
- hostsMax = 0
if "+" in cardinality:
hostsMin = int(cardinality[:-1])
hostsMax = sys.maxint
@@ -232,54 +78,13 @@ class HDP206StackAdvisor(StackAdvisor):
for host in nonUsedHostsList:
items.append( { "type": 'host-component', "level": 'ERROR', "message": 'Host is not used', "host": str(host) } )
- return validations
- pass
-
- def isNotValuable(self, component):
- componentName = component["StackServiceComponents"]["component_name"]
- service = ['JOURNALNODE', 'ZKFC', 'GANGLIA_MONITOR']
- return componentName in service
+ return items
- def recommendConfigurations(self, services, hosts):
- stackName = services["Versions"]["stack_name"]
- stackVersion = services["Versions"]["stack_version"]
- hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
- servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
- components = [component["StackServiceComponents"]["component_name"]
- for service in services["services"]
- for component in service["components"]]
-
- clusterData = self.getClusterData(servicesList, hosts, components)
-
- recommendations = {
- "Versions": {"stack_name": stackName, "stack_version": stackVersion},
- "hosts": hostsList,
- "services": servicesList,
- "recommendations": {
- "blueprint": {
- "configurations": {},
- "host_groups": []
- },
- "blueprint_cluster_binding": {
- "host_groups": []
- }
- }
- }
-
- configurations = recommendations["recommendations"]["blueprint"]["configurations"]
-
- for service in servicesList:
- calculation = self.recommendServiceConfigurations(service)
- if calculation is not None:
- calculation(configurations, clusterData)
-
- return recommendations
-
- def recommendServiceConfigurations(self, service):
+ def getServiceConfiguratorDict(self):
return {
"YARN": self.recommendYARNConfigurations,
"MAPREDUCE2": self.recommendMapReduce2Configurations
- }.get(service, None)
+ }
def putProperty(self, config, configType):
config[configType] = {"properties": {}}
@@ -379,17 +184,9 @@ class HDP206StackAdvisor(StackAdvisor):
return cluster
-
- def validateConfigurations(self, services, hosts):
+ def getConfigurationsValidationItems(self, services, hosts):
"""Returns array of Validation objects about issues with configuration values provided in services"""
- stackName = services["Versions"]["stack_name"]
- stackVersion = services["Versions"]["stack_version"]
-
- validations = {
- "Versions": {"stack_name": stackName, "stack_version": stackVersion},
- "items": [ ]
- }
- items = validations["items"]
+ items = []
recommendations = self.recommendConfigurations(services, hosts)
recommendedDefaults = recommendations["recommendations"]["blueprint"]["configurations"]
@@ -406,22 +203,24 @@ class HDP206StackAdvisor(StackAdvisor):
if siteProperties is not None:
resultItems = method(siteProperties, recommendedDefaults[siteName]["properties"])
items.extend(resultItems)
- return validations
- pass
+ return items
- def validateServiceConfigurations(self, serviceName):
+ def getServiceConfigurationValidators(self):
return {
"MAPREDUCE2": ["mapred-site", self.validateMapReduce2Configurations],
"YARN": ["yarn-site", self.validateYARNConfigurations]
- }.get(serviceName, None)
+ }
+
+ def validateServiceConfigurations(self, serviceName):
+ return self.getServiceConfigurationValidators().get(serviceName, None)
def toConfigurationValidationProblems(self, validationProblems, siteName):
result = []
for validationProblem in validationProblems:
validationItem = validationProblem.get("item", None)
if validationItem is not None:
- problem = { "type": 'configuration', "level": validationItem["level"], "message": validationItem["message"],
- "config-type": siteName, "config-name": validationProblem["config-name"] }
+ problem = {"type": 'configuration', "level": validationItem["level"], "message": validationItem["message"],
+ "config-type": siteName, "config-name": validationProblem["config-name"] }
result.append(problem)
return result
@@ -476,6 +275,36 @@ class HDP206StackAdvisor(StackAdvisor):
{"config-name": 'yarn.scheduler.maximum-allocation-mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.scheduler.maximum-allocation-mb')} ]
return self.toConfigurationValidationProblems(validationItems, "yarn-site")
+ def getMastersWithMultipleInstances(self):
+ return ['ZOOKEEPER_SERVER', 'HBASE_MASTER']
+
+ def getNotValuableComponents(self):
+ return ['JOURNALNODE', 'ZKFC', 'GANGLIA_MONITOR']
+
+ def getNotPreferableOnServerComponents(self):
+ return ['GANGLIA_SERVER', 'NAGIOS_SERVER']
+
+ def getCardinalitiesDict(self):
+ return {
+ 'ZOOKEEPER_SERVER': {"min": 3},
+ 'HBASE_MASTER': {"min": 1},
+ }
+
+ def selectionSchemes(self):
+ return {
+ 'NAMENODE': {"else": 0},
+ 'SECONDARY_NAMENODE': {"else": 1},
+ 'HBASE_MASTER': {6: 0, 31: 2, "else": 3},
+
+ 'HISTORYSERVER': {31: 1, "else": 2},
+ 'RESOURCEMANAGER': {31: 1, "else": 2},
+
+ 'OOZIE_SERVER': {6: 1, 31: 2, "else": 3},
+
+ 'HIVE_SERVER': {6: 1, 31: 2, "else": 4},
+ 'HIVE_METASTORE': {6: 1, 31: 2, "else": 4},
+ 'WEBHCAT_SERVER': {6: 1, 31: 2, "else": 4},
+ }
# Validation helper methods
def getSiteProperties(configurations, siteName):
@@ -519,45 +348,4 @@ def formatXmxSizeToBytes(value):
modifier == 't': 1024 * 1024 * 1024 * 1024,
modifier == 'p': 1024 * 1024 * 1024 * 1024 * 1024
}[1]
- return to_number(value) * m
-
-
-# Recommendation helper methods
-def isAlreadyPopulated(component):
- if component["StackServiceComponents"]["hostnames"] is not None:
- return len(component["StackServiceComponents"]["hostnames"]) > 0
- return False
-
-def isClient(component):
- return component["StackServiceComponents"]["component_category"] == 'CLIENT'
-
-def isSlave(component):
- componentName = component["StackServiceComponents"]["component_name"]
- isSlave = component["StackServiceComponents"]["component_category"] == 'SLAVE'
- return isSlave
-
-def isMaster(component):
- componentName = component["StackServiceComponents"]["component_name"]
- isMaster = component["StackServiceComponents"]["is_master"]
- return isMaster
-
-def isLocalHost(hostName):
- return socket.getfqdn(hostName) == socket.getfqdn()
-
-def isMasterWithMultipleInstances(component):
- componentName = component["StackServiceComponents"]["component_name"]
- masters = ['ZOOKEEPER_SERVER', 'HBASE_MASTER']
- return componentName in masters
-
-def defaultNoOfMasterHosts(component):
- componentName = component["StackServiceComponents"]["component_name"]
- return cardinality(componentName)[min]
-
-
-# Helper dictionaries
-def cardinality(componentName):
- return {
- 'ZOOKEEPER_SERVER': {min: 3},
- 'HBASE_MASTER': {min: 1},
- }.get(componentName, {min:1, max:1})
-
+ return to_number(value) * m
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/dc269bb1/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
index 98fee5d..bcf3f61 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
@@ -17,10 +17,6 @@ See the License for the specific language governing permissions and
limitations under the License.
"""
-import socket
-
-from stack_advisor import StackAdvisor
-
class HDP21StackAdvisor(HDP206StackAdvisor):
def recommendServiceConfigurations(self, service):
@@ -58,35 +54,45 @@ class HDP21StackAdvisor(HDP206StackAdvisor):
"-server -Xmx" + str(int(0.8 * clusterData["amMemory"]))
+ "m -Djava.net.preferIPv4Stack=true -XX:+UseNUMA -XX:+UseParallelGC")
- def isNotPreferableOnAmbariServerHost(self, component):
- componentName = component["StackServiceComponents"]["component_name"]
- service = ['STORM_UI_SERVER', 'DRPC_SERVER', 'STORM_REST_API', 'NIMBUS', 'GANGLIA_SERVER', 'NAGIOS_SERVER']
- return componentName in service
+ def getNotPreferableOnServerComponents(self):
+ return ['STORM_UI_SERVER', 'DRPC_SERVER', 'STORM_REST_API', 'NIMBUS', 'GANGLIA_SERVER', 'NAGIOS_SERVER']
- def isNotValuable(self, component):
- componentName = component["StackServiceComponents"]["component_name"]
- service = ['JOURNALNODE', 'ZKFC', 'GANGLIA_MONITOR', 'APP_TIMELINE_SERVER']
- return componentName in service
+ def getNotValuableComponents(self):
+ return ['JOURNALNODE', 'ZKFC', 'GANGLIA_MONITOR', 'APP_TIMELINE_SERVER']
- def selectionScheme(self, componentName):
- scheme = super(HDP21StackAdvisor, self).selectionScheme(componentName)
- if scheme is None:
- return {
+ def selectionSchemes(self):
+ return {
+ 'NAMENODE': {"else": 0},
+ 'SECONDARY_NAMENODE': {"else": 1},
+ 'HBASE_MASTER': {6: 0, 31: 2, "else": 3},
+
+ 'HISTORYSERVER': {31: 1, "else": 2},
+ 'RESOURCEMANAGER': {31: 1, "else": 2},
+
+ 'OOZIE_SERVER': {6: 1, 31: 2, "else": 3},
+
+ 'HIVE_SERVER': {6: 1, 31: 2, "else": 4},
+ 'HIVE_METASTORE': {6: 1, 31: 2, "else": 4},
+ 'WEBHCAT_SERVER': {6: 1, 31: 2, "else": 4},
+ }
+
+ def selectionScheme(self):
+ parentSchemes = super(HDP21StackAdvisor, self).selectionSchemes()
+ childSchemes = {
'APP_TIMELINE_SERVER': {31: 1, "else": 2},
'FALCON_SERVER': {6: 1, 31: 2, "else": 3}
- }.get(componentName, None)
- else:
- return scheme
-
- def validateServiceConfigurations(self, serviceName):
- validator = super(HDP21StackAdvisor, self).validateServiceConfigurations(serviceName)
- if validator is None:
- return {
- "HIVE": ["hive-site", self.validateHiveConfigurations],
- "TEZ": ["tez-site", self.validateTezConfigurations]
- }.get(serviceName, None)
- else:
- return validator
+ }
+ parentSchemes.update(childSchemes)
+ return parentSchemes
+
+ def getServiceConfigurationValidators(self):
+ parentValidators = super(HDP21StackAdvisor, self).getServiceConfigurationValidators()
+ childValidators = {
+ "HIVE": ["hive-site", self.validateHiveConfigurations],
+ "TEZ": ["tez-site", self.validateTezConfigurations]
+ }
+ parentValidators.update(childValidators)
+ return parentValidators
def validateHiveConfigurations(self, properties, recommendedDefaults):
validationItems = [ {"config-name": 'hive.tez.container.size', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'hive.tez.container.size')},
http://git-wip-us.apache.org/repos/asf/ambari/blob/dc269bb1/ambari-server/src/main/resources/stacks/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/stack_advisor.py b/ambari-server/src/main/resources/stacks/stack_advisor.py
index 5f66fff..dca2834 100644
--- a/ambari-server/src/main/resources/stacks/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/stack_advisor.py
@@ -17,6 +17,8 @@ See the License for the specific language governing permissions and
limitations under the License.
"""
+import socket
+
class StackAdvisor(object):
def recommendComponentLayout(self, services, hosts):
@@ -35,3 +37,272 @@ class StackAdvisor(object):
"""Returns array of Validation objects about issues with configuration values provided in services"""
pass
+class DefaultStackAdvisor(StackAdvisor):
+
+ def recommendComponentLayout(self, services, hosts):
+ """Returns Services object with hostnames array populated for components"""
+
+ stackName = services["Versions"]["stack_name"]
+ stackVersion = services["Versions"]["stack_version"]
+ hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
+ servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+
+ layoutRecommendations = self.provideLayout(services, hosts)
+
+ recommendations = {
+ "Versions": {"stack_name": stackName, "stack_version": stackVersion},
+ "hosts": hostsList,
+ "services": servicesList,
+ "recommendations": layoutRecommendations
+ }
+
+ return recommendations
+
+ def provideLayout(self, services, hosts):
+
+ recommendations = {
+ "blueprint": {
+ "host_groups": [ ]
+ },
+ "blueprint_cluster_binding": {
+ "host_groups": [ ]
+ }
+ }
+
+ hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
+
+ hostsComponentsMap = {}
+ for hostName in hostsList:
+ if hostName not in hostsComponentsMap:
+ hostsComponentsMap[hostName] = []
+
+ #extend 'hostsComponentsMap' with MASTER components
+ for service in services["services"]:
+ masterComponents = [component for component in service["components"] if self.isMaster(component)]
+ for component in masterComponents:
+ componentName = component["StackServiceComponents"]["component_name"]
+
+ if self.isAlreadyPopulated(component):
+ hostsForComponent = component["StackServiceComponents"]["hostnames"]
+ else:
+ availableHosts = hostsList
+ if len(hostsList) > 1 and self.isNotPreferableOnAmbariServerHost(component):
+ availableHosts = [hostName for hostName in hostsList if not self.isLocalHost(hostName)]
+
+ if self.isMasterWithMultipleInstances(component):
+ hostsCount = self.defaultNoOfMasterHosts(component)
+ if hostsCount > 1: # get first 'hostsCount' available hosts
+ if len(availableHosts) < hostsCount:
+ hostsCount = len(availableHosts)
+ hostsForComponent = availableHosts[:hostsCount]
+ else:
+ hostsForComponent = [self.getHostForComponent(component, availableHosts, hostsComponentsMap)]
+ else:
+ hostsForComponent = [self.getHostForComponent(component, availableHosts, hostsComponentsMap)]
+
+ #extend 'hostsComponentsMap' with 'hostsForComponent'
+ for hostName in hostsForComponent:
+ hostsComponentsMap[hostName].append( { "name":componentName } )
+
+ #extend 'hostsComponentsMap' with Slave and Client Components
+ componentsListList = [service["components"] for service in services["services"]]
+ componentsList = [item for sublist in componentsListList for item in sublist]
+ usedHostsListList = [component["StackServiceComponents"]["hostnames"] for component in componentsList if not self.isNotValuable(component)]
+ utilizedHosts = [item for sublist in usedHostsListList for item in sublist]
+ freeHosts = [hostName for hostName in hostsList if hostName not in utilizedHosts]
+
+ for service in services["services"]:
+ slaveClientComponents = [component for component in service["components"]
+ if self.isSlave(component) or self.isClient(component)]
+ for component in slaveClientComponents:
+ componentName = component["StackServiceComponents"]["component_name"]
+
+ if self.isAlreadyPopulated(component):
+ hostsForComponent = component["StackServiceComponents"]["hostnames"]
+ elif component["StackServiceComponents"]["cardinality"] == "ALL":
+ hostsForComponent = hostsList
+ else:
+ if len(freeHosts) == 0:
+ hostsForComponent = hostsList[-1:]
+ else: # len(freeHosts) >= 1
+ hostsForComponent = freeHosts
+ if self.isClient(component):
+ hostsForComponent = freeHosts[0:1]
+
+ #extend 'hostsComponentsMap' with 'hostsForComponent'
+ for hostName in hostsForComponent:
+ if hostName not in hostsComponentsMap:
+ hostsComponentsMap[hostName] = []
+ hostsComponentsMap[hostName].append( { "name": componentName } )
+
+ #prepare 'host-group's from 'hostsComponentsMap'
+ host_groups = recommendations["blueprint"]["host_groups"]
+ bindings = recommendations["blueprint_cluster_binding"]["host_groups"]
+ index = 0
+ for key in hostsComponentsMap.keys():
+ index += 1
+ host_group_name = "host-group-{0}".format(index)
+ host_groups.append( { "name": host_group_name, "components": hostsComponentsMap[key] } )
+ bindings.append( { "name": host_group_name, "hosts": [{ "fqdn": socket.getfqdn(key) }] } )
+
+ return recommendations
+ pass
+
+ def prepareValidationResponse(self, services, validationItems):
+ """Returns array of Validation objects about issues with hostnames components assigned to"""
+ stackName = services["Versions"]["stack_name"]
+ stackVersion = services["Versions"]["stack_version"]
+
+ validations = {
+ "Versions": {"stack_name": stackName, "stack_version": stackVersion},
+ "items": validationItems
+ }
+
+ return validations
+
+ def validateComponentLayout(self, services, hosts):
+ """Returns array of Validation objects about issues with hostnames components assigned to"""
+ validationItems = self.getLayoutValidationItems(services, hosts)
+ return self.prepareValidationResponse(services, validationItems)
+
+ def validateConfigurations(self, services, hosts):
+ """Returns array of Validation objects about issues with hostnames components assigned to"""
+ validationItems = self.getConfigurationsValidationItems(services, hosts)
+ return self.prepareValidationResponse(services, validationItems)
+
+ def getLayoutValidationItems(self, services, hosts):
+ return []
+
+ def getClusterData(self, servicesList, hosts, components):
+ pass
+
+ def getConfigurationsValidationItems(self, services, hosts):
+ return []
+
+ def recommendConfigurations(self, services, hosts):
+ stackName = services["Versions"]["stack_name"]
+ stackVersion = services["Versions"]["stack_version"]
+ hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
+ servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+ components = [component["StackServiceComponents"]["component_name"]
+ for service in services["services"]
+ for component in service["components"]]
+
+ clusterData = self.getClusterData(servicesList, hosts, components)
+
+ recommendations = {
+ "Versions": {"stack_name": stackName, "stack_version": stackVersion},
+ "hosts": hostsList,
+ "services": servicesList,
+ "recommendations": {
+ "blueprint": {
+ "configurations": {},
+ "host_groups": []
+ },
+ "blueprint_cluster_binding": {
+ "host_groups": []
+ }
+ }
+ }
+
+ configurations = recommendations["recommendations"]["blueprint"]["configurations"]
+
+ for service in servicesList:
+ calculation = self.recommendServiceConfigurations(service)
+ if calculation is not None:
+ calculation(configurations, clusterData)
+
+ return recommendations
+
+ def recommendServiceConfigurations(self, service):
+ return self.getServiceConfiguratorDict().get(service, None)
+
+ def getServiceConfiguratorDict(self):
+ return {}
+
+ # Recommendation helper methods
+ def isAlreadyPopulated(self, component):
+ hostnames = self.getComponentAttribute(component, "hostnames")
+ if hostnames is not None:
+ return len(hostnames) > 0
+ return False
+
+ def isClient(self, component):
+ return self.getComponentAttribute(component, "component_category") == 'CLIENT'
+
+ def isSlave(self, component):
+ return self.getComponentAttribute(component, "component_category") == 'SLAVE'
+
+ def isMaster(self, component):
+ return self.getComponentAttribute(component, "is_master")
+
+ def getComponentAttribute(self, component, attribute):
+ serviceComponent = component.get("StackServiceComponents", None)
+ if serviceComponent is None:
+ return None
+ return serviceComponent.get(attribute, None)
+
+ def isLocalHost(self, hostName):
+ return socket.getfqdn(hostName) == socket.getfqdn()
+
+ def isMasterWithMultipleInstances(self, component):
+ componentName = self.getComponentName(component)
+ masters = self.getMastersWithMultipleInstances()
+ return componentName in masters
+
+ def isNotValuable(self, component):
+ componentName = self.getComponentName(component)
+ service = self.getNotValuableComponents()
+ return componentName in service
+
+ def defaultNoOfMasterHosts(self, component):
+ componentName = self.getComponentName(component)
+ return self.cardinality(componentName)["min"]
+
+ # Helper dictionaries
+ def cardinality(self, componentName):
+ return self.getCardinalitiesDict().get(componentName, {"min": 1, "max": 1})
+
+ def getHostForComponent(self, component, hostsList, hostsComponentsMap):
+ componentName = self.getComponentName(component)
+
+ if len(hostsList) == 1:
+ return hostsList[0]
+ else:
+ scheme = self.defineSelectionScheme(componentName)
+ if scheme is not None:
+ for key in scheme.keys():
+ if isinstance(key, ( int, long )):
+ if len(hostsList) < key:
+ return hostsList[scheme[key]]
+ return self.getLeastOccupiedHost(hostsList, hostsComponentsMap)
+
+ def getLeastOccupiedHost(self, hostsList, hostComponentsMap):
+ hostOccupations = dict((host, len(components)) for host, components in hostComponentsMap.iteritems() if host in hostsList)
+ return min(hostOccupations, key=hostOccupations.get)
+
+ def defineSelectionScheme(self, componentName):
+ return self.selectionSchemes().get(componentName, None)
+
+ def getComponentName(self, component):
+ return self.getComponentAttribute(component, "component_name")
+
+ def isNotPreferableOnAmbariServerHost(self, component):
+ componentName = self.getComponentName(component)
+ service = self.getNotPreferableOnServerComponents()
+ return componentName in service
+
+ def getMastersWithMultipleInstances(self):
+ return []
+
+ def getNotValuableComponents(self):
+ return []
+
+ def getNotPreferableOnServerComponents(self):
+ return []
+
+ def getCardinalitiesDict(self):
+ return {}
+
+ def selectionSchemes(self):
+ return {}
http://git-wip-us.apache.org/repos/asf/ambari/blob/dc269bb1/ambari-server/src/test/python/TestStackAdvisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/TestStackAdvisor.py b/ambari-server/src/test/python/TestStackAdvisor.py
index 5a1a242..3f56816 100644
--- a/ambari-server/src/test/python/TestStackAdvisor.py
+++ b/ambari-server/src/test/python/TestStackAdvisor.py
@@ -38,40 +38,41 @@ class TestStackAdvisorInitialization(TestCase):
instantiate_stack_advisor_method = getattr(self.stack_advisor, instantiate_stack_advisor_method_name)
stack_advisor = instantiate_stack_advisor_method("XYZ", "1.0.1", ["1.0.0"])
self.assertEquals("XYZ101StackAdvisor", stack_advisor.__class__.__name__)
- services = {"Versions":
- {
- "stack_name":"XYZ",
- "stack_version":"1.0.1"
- },
- "services":[
- {
- "StackServices":{
- "service_name":"YARN"
- },
- "components":[
- {
- "StackServiceComponents": {
- "component_name": "RESOURCEMANAGER"
- }
- },
- {
- "StackServiceComponents": {
- "component_name": "APP_TIMELINE_SERVER"
- }
- },
- {
- "StackServiceComponents": {
- "component_name":"YARN_CLIENT"
- }
- },
- {
- "StackServiceComponents": {
- "component_name": "NODEMANAGER"
- }
- }
- ]
- }
- ]
+ services = {
+ "Versions":
+ {
+ "stack_name":"XYZ",
+ "stack_version":"1.0.1"
+ },
+ "services":[
+ {
+ "StackServices":{
+ "service_name":"YARN"
+ },
+ "components":[
+ {
+ "StackServiceComponents": {
+ "component_name": "RESOURCEMANAGER"
+ }
+ },
+ {
+ "StackServiceComponents": {
+ "component_name": "APP_TIMELINE_SERVER"
+ }
+ },
+ {
+ "StackServiceComponents": {
+ "component_name":"YARN_CLIENT"
+ }
+ },
+ {
+ "StackServiceComponents": {
+ "component_name": "NODEMANAGER"
+ }
+ }
+ ]
+ }
+ ]
}
hosts= {
"items": [
@@ -84,10 +85,258 @@ class TestStackAdvisorInitialization(TestCase):
'''Check that value is populated from child class, not parent'''
self.assertEquals("-Xmx101m", yarn_configs["yarn.nodemanager.resource.memory-mb"])
- def test_stackAdvisorSuperClassIsFoundAndReturnedAsDefaultImpl(self):
+ def test_stackAdvisorDefaultImpl(self):
instantiate_stack_advisor_method_name = 'instantiateStackAdvisor'
instantiate_stack_advisor_method = getattr(self.stack_advisor, instantiate_stack_advisor_method_name)
'''Not existent stack - to return default implementation'''
default_stack_advisor = instantiate_stack_advisor_method("HDP1", "2.0.6", [])
- self.assertEquals("StackAdvisor", default_stack_advisor.__class__.__name__)
-
+ self.assertEquals("DefaultStackAdvisor", default_stack_advisor.__class__.__name__)
+ services = {
+ "Versions":
+ {
+ "stack_name":"HDP1",
+ "stack_version":"2.0.6"
+ },
+ "services" : [
+ {
+ "StackServices" : {
+ "service_name" : "GANGLIA",
+ "service_version" : "3.5.0",
+ },
+ "components" : [
+ {
+ "StackServiceComponents" : {
+ "cardinality" : "ALL",
+ "component_name" : "GANGLIA_MONITOR",
+ "is_master" : False,
+ "hostnames" : [ ]
+ }
+ },
+ {
+ "StackServiceComponents" : {
+ "cardinality" : "1",
+ "component_name" : "GANGLIA_SERVER",
+ "is_master" : True,
+ "hostnames" : [ ]
+ }
+ }
+ ]
+ },
+ {
+ "StackServices" : {
+ "service_name" : "HBASE",
+ "service_version" : "0.98.0.2.1"
+ },
+ "components" : [
+ {
+ "StackServiceComponents" : {
+ "cardinality" : "1+",
+ "component_name" : "HBASE_CLIENT",
+ "is_master" : False,
+ "hostnames" : [ ]
+ }
+ },
+ {
+ "StackServiceComponents" : {
+ "cardinality" : "1+",
+ "component_name" : "HBASE_MASTER",
+ "is_master" : True,
+ "hostnames" : [ ]
+ }
+ },
+ {
+ "StackServiceComponents" : {
+ "cardinality" : "1+",
+ "component_name" : "HBASE_REGIONSERVER",
+ "is_master" : False,
+ "hostnames" : [ ]
+ }
+ }
+ ]
+ },
+ {
+ "StackServices" : {
+ "service_name" : "HDFS",
+ "service_version" : "2.4.0.2.1"
+ },
+ "components" : [
+ {
+ "StackServiceComponents" : {
+ "cardinality" : "1+",
+ "component_name" : "DATANODE",
+ "is_master" : False,
+ "hostnames" : [ ]
+ }
+ }, {
+ "StackServiceComponents" : {
+ "cardinality" : "1+",
+ "component_name" : "HDFS_CLIENT",
+ "is_master" : False,
+ "hostnames" : [ ]
+ }
+ }, {
+ "StackServiceComponents" : {
+ "cardinality" : "0+",
+ "component_name" : "JOURNALNODE",
+ "is_master" : False,
+ "hostnames" : [ ]
+ }
+ },
+ {
+ "StackServiceComponents" : {
+ "cardinality" : "1-2",
+ "component_name" : "NAMENODE",
+ "is_master" : True,
+ "hostnames" : [ ]
+ }
+ },
+ {
+ "StackServiceComponents" : {
+ "cardinality" : "1",
+ "component_name" : "SECONDARY_NAMENODE",
+ "is_master" : True,
+ "hostnames" : [ ]
+ }
+ },
+ {
+ "StackServiceComponents" : {
+ "cardinality" : "0+",
+ "component_name" : "ZKFC",
+ "is_master" : False,
+ "hostnames" : [ ]
+ }
+ }
+ ]
+ },
+ {
+ "StackServices" : {
+ "service_name" : "PIG",
+ "service_version" : "0.12.1.2.1"
+ },
+ "components" : [
+ {
+ "StackServiceComponents" : {
+ "cardinality" : "0+",
+ "component_name" : "PIG",
+ "is_master" : False,
+ "hostnames" : [ ]
+ }
+ }
+ ]
+ },
+ {
+ "StackServices" : {
+ "service_name" : "TEZ",
+ "service_version" : "0.4.0.2.1"
+ },
+ "components" : [
+ {
+ "StackServiceComponents" : {
+ "cardinality" : "0+",
+ "component_name" : "TEZ_CLIENT",
+ "is_master" : False,
+ "hostnames" : [ ]
+ }
+ }
+ ]
+ },
+ {
+ "StackServices" : {
+ "service_name" : "ZOOKEEPER",
+ "service_version" : "3.4.5.2.1",
+ },
+ "components" : [
+ {
+ "StackServiceComponents" : {
+ "cardinality" : "1+",
+ "component_category" : "CLIENT",
+ "component_name" : "ZOOKEEPER_CLIENT",
+ "is_master" : False,
+ "hostnames" : [ ]
+ }
+ },
+ {
+ "StackServiceComponents" : {
+ "cardinality" : "1+",
+ "component_name" : "ZOOKEEPER_SERVER",
+ "is_master" : True,
+ "hostnames" : [ ]
+ }
+ }
+ ]
+ }
+ ],
+ "configurations" : {}
+ }
+ hosts= {
+ "items": [
+ {"Hosts": {"host_name": "host1"}},
+ {"Hosts": {"host_name": "host2"}}
+ ]
+ }
+ actualValidateConfigResponse = default_stack_advisor.validateConfigurations(services, hosts)
+ actualValidateLayoutResponse = default_stack_advisor.validateComponentLayout(services, hosts)
+ expectedValidationResponse = {
+ "Versions": {"stack_name": "HDP1", "stack_version": "2.0.6"},
+ "items": []
+ }
+ self.assertEquals(actualValidateConfigResponse, expectedValidationResponse)
+ self.assertEquals(actualValidateLayoutResponse, expectedValidationResponse)
+ actualRecommendConfigResponse = default_stack_advisor.recommendConfigurations(services, hosts)
+ expectedRecommendConfigResponse = {
+ "Versions": {"stack_name": "HDP1", "stack_version": "2.0.6"},
+ "hosts": ["host1", "host2"],
+ "services": ['GANGLIA', 'HBASE', 'HDFS', 'PIG', 'TEZ', 'ZOOKEEPER'],
+ "recommendations": {
+ "blueprint": {
+ "configurations": {},
+ "host_groups": []
+ },
+ "blueprint_cluster_binding": {
+ "host_groups": []
+ }
+ }
+ }
+ self.assertEquals(actualRecommendConfigResponse, expectedRecommendConfigResponse)
+ actualRecommendLayoutResponse = default_stack_advisor.recommendComponentLayout(services, hosts)
+ expectedRecommendLayoutResponse = {
+ "Versions": {"stack_name": "HDP1", "stack_version": "2.0.6"},
+ "hosts": ["host1", "host2"],
+ "services": ['GANGLIA', 'HBASE', 'HDFS', 'PIG', 'TEZ', 'ZOOKEEPER'],
+ "recommendations": {
+ "blueprint": {
+ "host_groups": [
+ {
+ "name": "host-group-1",
+ "components": [
+ {"name": "GANGLIA_SERVER"},
+ {"name": "NAMENODE"},
+ {"name": "ZOOKEEPER_SERVER"}
+ ]
+ },
+ {
+ "name": "host-group-2",
+ "components": [
+ {"name": "HBASE_MASTER"},
+ {"name": "SECONDARY_NAMENODE"},
+ {"name": "ZOOKEEPER_CLIENT"}
+ ]
+ }
+ ]
+ },
+ "blueprint_cluster_binding":
+ {
+ "host_groups": [
+ {
+ "name": "host-group-1",
+ "hosts": [{"fqdn": "host2"}]
+ },
+ {
+ "name": "host-group-2",
+ "hosts": [{"fqdn": "host1"}]
+ }
+ ]
+ }
+ }
+ }
+ self.assertEquals(actualRecommendLayoutResponse, expectedRecommendLayoutResponse)
http://git-wip-us.apache.org/repos/asf/ambari/blob/dc269bb1/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
index b1b9f7a..aa86242 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
@@ -35,7 +35,7 @@ class TestHDP206StackAdvisor(TestCase):
stack_advisor_impl = imp.load_module('stack_advisor_impl', fp, hdp206StackAdvisorPath, ('.py', 'rb', imp.PY_SOURCE))
clazz = getattr(stack_advisor_impl, hdp206StackAdvisorClassName)
self.stackAdvisor = clazz()
-
+
def test_recommendationCardinalityALL(self):
servicesInfo = [
{