You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by rl...@apache.org on 2017/06/19 20:16:42 UTC
[20/25] ambari git commit: AMBARI-20122 - Stack advisor needs to
recommend dependency for slaves and masters
AMBARI-20122 - Stack advisor needs to recommend dependency for slaves and masters
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/927d5c83
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/927d5c83
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/927d5c83
Branch: refs/heads/branch-feature-AMBARI-20859
Commit: 927d5c8340e5a22cd2fb8b8a2d14956b19810b93
Parents: e8e9781
Author: Tim Thorpe <tt...@apache.org>
Authored: Mon Jun 19 09:00:20 2017 -0700
Committer: Tim Thorpe <tt...@apache.org>
Committed: Mon Jun 19 09:00:20 2017 -0700
----------------------------------------------------------------------
.../src/main/resources/stacks/stack_advisor.py | 116 +++++++++++++-
.../stacks/2.0.6/common/test_stack_advisor.py | 153 +++++++++++++++++++
2 files changed, 264 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/927d5c83/ambari-server/src/main/resources/stacks/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/stack_advisor.py b/ambari-server/src/main/resources/stacks/stack_advisor.py
index 4a81dc6..3a39a34 100644
--- a/ambari-server/src/main/resources/stacks/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/stack_advisor.py
@@ -756,15 +756,19 @@ class DefaultStackAdvisor(StackAdvisor):
if hostName not in hostsComponentsMap:
hostsComponentsMap[hostName] = []
+ #Sort the services so that the dependent services will be processed before those that depend on them.
+ sortedServices = self.getServicesSortedByDependencies(services)
#extend hostsComponentsMap' with MASTER components
- for service in services["services"]:
+ for service in sortedServices:
masterComponents = [component for component in service["components"] if self.isMasterComponent(component)]
serviceName = service["StackServices"]["service_name"]
serviceAdvisor = self.getServiceAdvisor(serviceName)
for component in masterComponents:
componentName = component["StackServiceComponents"]["component_name"]
advisor = serviceAdvisor if serviceAdvisor is not None else self
- hostsForComponent = advisor.getHostsForMasterComponent(services, hosts, component, hostsList)
+ #Filter the hosts such that only hosts that meet the dependencies are included (if possible)
+ filteredHosts = self.getFilteredHostsBasedOnDependencies(services, component, hostsList, hostsComponentsMap)
+ hostsForComponent = advisor.getHostsForMasterComponent(services, hosts, component, filteredHosts)
#extend 'hostsComponentsMap' with 'hostsForComponent'
for hostName in hostsForComponent:
@@ -778,7 +782,7 @@ class DefaultStackAdvisor(StackAdvisor):
utilizedHosts = [item for sublist in usedHostsListList for item in sublist]
freeHosts = [hostName for hostName in hostsList if hostName not in utilizedHosts]
- for service in services["services"]:
+ for service in sortedServices:
slaveClientComponents = [component for component in service["components"]
if self.isSlaveComponent(component) or self.isClientComponent(component)]
serviceName = service["StackServices"]["service_name"]
@@ -786,7 +790,10 @@ class DefaultStackAdvisor(StackAdvisor):
for component in slaveClientComponents:
componentName = component["StackServiceComponents"]["component_name"]
advisor = serviceAdvisor if serviceAdvisor is not None else self
- hostsForComponent = advisor.getHostsForSlaveComponent(services, hosts, component, hostsList, freeHosts)
+ #Filter the hosts and free hosts such that only hosts that meet the dependencies are included (if possible)
+ filteredHosts = self.getFilteredHostsBasedOnDependencies(services, component, hostsList, hostsComponentsMap)
+ filteredFreeHosts = self.filterList(freeHosts, filteredHosts)
+ hostsForComponent = advisor.getHostsForSlaveComponent(services, hosts, component, filteredHosts, filteredFreeHosts)
#extend 'hostsComponentsMap' with 'hostsForComponent'
for hostName in hostsForComponent:
@@ -796,7 +803,7 @@ class DefaultStackAdvisor(StackAdvisor):
hostsComponentsMap[hostName].append( { "name": componentName } )
#colocate custom services
- for service in services["services"]:
+ for service in sortedServices:
serviceName = service["StackServices"]["service_name"]
serviceAdvisor = self.getServiceAdvisor(serviceName)
if serviceAdvisor is not None:
@@ -866,6 +873,105 @@ class DefaultStackAdvisor(StackAdvisor):
return hostsForComponent
+ def getServicesSortedByDependencies(self, services):
+ """
+ Sorts the services based on their dependencies. This is limited to non-conditional host scope dependencies.
+ Services with no dependencies will go first. Services with dependencies will go after the services they are dependent on.
+ If there are circular dependencies, the services will go in the order in which they were processed.
+ """
+ processedServices = []
+ sortedServices = []
+
+ for service in services["services"]:
+ self.sortServicesByDependencies(services, service, processedServices, sortedServices)
+
+ return sortedServices
+
+ def sortServicesByDependencies(self, services, service, processedServices, sortedServices):
+ """
+ Sorts the services based on their dependencies. This is limited to non-conditional host scope dependencies.
+ Services with no dependencies will go first. Services with dependencies will go after the services they are dependent on.
+ If there are circular dependencies, the services will go in the order in which they were processed.
+ """
+ if service is None or service in processedServices:
+ return
+
+ processedServices.append(service)
+
+ components = [] if "components" not in service else service["components"]
+ for component in components:
+ dependencies = [] if "dependencies" not in component else component['dependencies']
+ for dependency in dependencies:
+ # accounts only for dependencies that are not conditional
+ conditionsPresent = "conditions" in dependency["Dependencies"] and dependency["Dependencies"]["conditions"]
+ scope = "cluster" if "scope" not in dependency["Dependencies"] else dependency["Dependencies"]["scope"]
+ if not conditionsPresent and scope == "host":
+ componentName = component["StackServiceComponents"]["component_name"]
+ requiredComponentName = dependency["Dependencies"]["component_name"]
+ requiredService = self.getServiceForComponentName(services, requiredComponentName)
+ self.sortServicesByDependencies(services, requiredService, processedServices, sortedServices)
+
+ sortedServices.append(service)
+
+ def getFilteredHostsBasedOnDependencies(self, services, component, hostsList, hostsComponentsMap):
+ """
+ Returns a list of hosts that only includes the ones which have all host scope dependencies already assigned to them.
+ If an empty list would be returned, instead the full list of hosts are returned.
+ In that case, we can't possibly return a valid recommended layout so we will at least return a fully filled layout.
+ """
+ removeHosts = []
+ dependencies = [] if "dependencies" not in component else component['dependencies']
+ for dependency in dependencies:
+ # accounts only for dependencies that are not conditional
+ conditionsPresent = "conditions" in dependency["Dependencies"] and dependency["Dependencies"]["conditions"]
+ if not conditionsPresent:
+ componentName = component["StackServiceComponents"]["component_name"]
+ requiredComponentName = dependency["Dependencies"]["component_name"]
+ requiredComponent = self.getRequiredComponent(services, requiredComponentName)
+
+ # We only deal with "host" scope.
+ if (requiredComponent is not None) and (requiredComponent["component_category"] != "CLIENT"):
+ scope = "cluster" if "scope" not in dependency["Dependencies"] else dependency["Dependencies"]["scope"]
+ if scope == "host":
+ for host, hostComponents in hostsComponentsMap.iteritems():
+ isRequiredIncluded = False
+ for component in hostComponents:
+ currentComponentName = None if "name" not in component else component["name"]
+ if requiredComponentName == currentComponentName:
+ isRequiredIncluded = True
+ if not isRequiredIncluded:
+ removeHosts.append(host)
+
+ filteredHostsList = []
+ for host in hostsList:
+ if host not in removeHosts:
+ filteredHostsList.append(host)
+ return filteredHostsList
+
+ def filterList(self, list, filter):
+ """
+ Returns the union of the two lists passed in (list and filter params).
+ """
+ filteredList = []
+ for item in list:
+ if item in filter:
+ filteredList.append(item)
+ return filteredList
+
+ def getServiceForComponentName(self, services, componentName):
+ """
+ Return service for component name
+
+ :type services dict
+ :type componentName str
+ """
+ for service in services["services"]:
+ for component in service["components"]:
+ if self.getComponentName(component) == componentName:
+ return service
+
+ return None
+
def isComponentUsingCardinalityForLayout(self, componentName):
return False
http://git-wip-us.apache.org/repos/asf/ambari/blob/927d5c83/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
index 41c57f6..b6f1965 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
@@ -191,6 +191,159 @@ class TestHDP206StackAdvisor(TestCase):
]
self.assertValidationResult(expectedItems, result)
+
+ def test_handleComponentDependencies(self):
+ services = {
+ "Versions":
+ {
+ "stack_name":"HDP",
+ "stack_version":"2.0.6"
+ },
+ "services" : [
+ {
+ "StackServices" : {
+ "service_name" : "HDFS",
+ "service_version" : "2.0.6",
+ },
+ "components": [
+ {
+ "StackServiceComponents": {
+ "stack_version": "2.0.6",
+ "stack_name": "HDP",
+ "component_category": "MASTER",
+ "is_client": False,
+ "is_master": True,
+ "service_name": "HDFS",
+ "cardinality": "1-2",
+ "hostnames": [],
+ "component_name": "NAMENODE",
+ "display_name": "NameNode"
+ },
+ "dependencies": [
+ {
+ "Dependencies": {
+ "stack_name": "HDP",
+ "stack_version": "2.0.6",
+ "scope": "cluster",
+ "conditions": [
+ {
+ "configType": "hdfs-site",
+ "property": "dfs.nameservices",
+ "type": "PropertyExists",
+ }
+ ],
+ "dependent_service_name": "HDFS",
+ "dependent_component_name": "NAMENODE",
+ "component_name": "ZOOKEEPER_SERVER"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "StackServices" : {
+ "service_name" : "ZOOKEEPER",
+ "service_version" : "2.0.6",
+ },
+ "components": [
+ {
+ "StackServiceComponents": {
+ "stack_version": "2.0.6",
+ "stack_name": "HDP",
+ "component_category": "MASTER",
+ "is_client": False,
+ "is_master": True,
+ "service_name": "HDFS",
+ "cardinality": "1-2",
+ "hostnames": [],
+ "component_name": "ZOOKEEPER_SERVER",
+ "display_name": "ZooKeeper Server"
+ },
+ "dependencies": []
+ }
+ ]
+ }
+ ]
+ }
+
+ nameNodeDependencies = services["services"][0]["components"][0]["dependencies"][0]["Dependencies"]
+
+ # Tests for master component with dependencies
+
+ hosts = self.prepareHosts(["c6401.ambari.apache.org", "c6402.ambari.apache.org", "c6403.ambari.apache.org", "c6404.ambari.apache.org"])
+ services["services"][1]["components"][0]["StackServiceComponents"]["hostnames"] = ["c6402.ambari.apache.org", "c6403.ambari.apache.org"]
+ recommendations = self.stackAdvisor.createComponentLayoutRecommendations(services, hosts)
+ # Assert that dependencies are ignored when there are conditions and cluster scope
+ self.assertEquals(recommendations['blueprint']['host_groups'][3]['components'][0]['name'], 'NAMENODE')
+ self.assertEquals(len(recommendations['blueprint']['host_groups'][3]['components']), 1)
+
+ nameNodeDependencies["scope"] = "host"
+ recommendations = self.stackAdvisor.createComponentLayoutRecommendations(services, hosts)
+ # Assert that dependencies are ignored when there are conditions (even for host scope)
+ self.assertEquals(recommendations['blueprint']['host_groups'][3]['components'][0]['name'], 'NAMENODE')
+ self.assertEquals(len(recommendations['blueprint']['host_groups'][3]['components']), 1)
+
+ nameNodeDependencies["scope"] = "cluster"
+ originalConditions = nameNodeDependencies["conditions"]
+ nameNodeDependencies["conditions"] = []
+ recommendations = self.stackAdvisor.createComponentLayoutRecommendations(services, hosts)
+ # Assert that dependencies are ignored when scope is cluster
+ self.assertEquals(recommendations['blueprint']['host_groups'][3]['components'][0]['name'], 'NAMENODE')
+ self.assertEquals(len(recommendations['blueprint']['host_groups'][3]['components']), 1)
+
+ nameNodeDependencies["scope"] = "host"
+ recommendations = self.stackAdvisor.createComponentLayoutRecommendations(services, hosts)
+ # Assert that dependencies are enforced for host scope without conditions
+ #self.assertEquals(recommendations, "")
+ self.assertEquals(len(recommendations['blueprint']['host_groups'][0]['components']), 2)
+
+ services["services"][1]["components"][0]["StackServiceComponents"]["is_master"] = False
+ services["services"][1]["components"][0]["StackServiceComponents"]["component_category"] = "CLIENT"
+ recommendations = self.stackAdvisor.createComponentLayoutRecommendations(services, hosts)
+ # Assert that dependencies are ignored when depending on client components
+ self.assertEquals(recommendations['blueprint']['host_groups'][3]['components'][0]['name'], 'NAMENODE')
+ self.assertEquals(len(recommendations['blueprint']['host_groups'][3]['components']), 1)
+
+ # Tests for slave component with dependencies
+ services["services"][0]["components"][0]["StackServiceComponents"]["component_category"] = "SLAVE"
+ services["services"][0]["components"][0]["StackServiceComponents"]["is_master"] = False
+ services["services"][1]["components"][0]["StackServiceComponents"]["component_category"] = "MASTER"
+ services["services"][1]["components"][0]["StackServiceComponents"]["is_master"] = True
+
+ nameNodeDependencies["scope"] = "cluster"
+ nameNodeDependencies["conditions"] = originalConditions
+ recommendations = self.stackAdvisor.createComponentLayoutRecommendations(services, hosts)
+ # Assert that dependencies are ignored when there are conditions and cluster scope
+ self.assertEquals(recommendations['blueprint']['host_groups'][2]['components'][0]['name'], 'NAMENODE')
+ self.assertEquals(len(recommendations['blueprint']['host_groups'][2]['components']), 1)
+ self.assertEquals(recommendations['blueprint']['host_groups'][3]['components'][0]['name'], 'NAMENODE')
+ self.assertEquals(len(recommendations['blueprint']['host_groups'][3]['components']), 1)
+
+ nameNodeDependencies["scope"] = "host"
+ recommendations = self.stackAdvisor.createComponentLayoutRecommendations(services, hosts)
+ # Assert that dependencies are ignored when there are conditions (even for host scope)
+ self.assertEquals(recommendations['blueprint']['host_groups'][2]['components'][0]['name'], 'NAMENODE')
+ self.assertEquals(len(recommendations['blueprint']['host_groups'][2]['components']), 1)
+ self.assertEquals(recommendations['blueprint']['host_groups'][3]['components'][0]['name'], 'NAMENODE')
+ self.assertEquals(len(recommendations['blueprint']['host_groups'][3]['components']), 1)
+
+ nameNodeDependencies["scope"] = "cluster"
+ nameNodeDependencies["conditions"] = []
+ recommendations = self.stackAdvisor.createComponentLayoutRecommendations(services, hosts)
+ # Assert that dependencies are ignored when scope is cluster
+ self.assertEquals(recommendations['blueprint']['host_groups'][2]['components'][0]['name'], 'NAMENODE')
+ self.assertEquals(len(recommendations['blueprint']['host_groups'][2]['components']), 1)
+ self.assertEquals(recommendations['blueprint']['host_groups'][3]['components'][0]['name'], 'NAMENODE')
+ self.assertEquals(len(recommendations['blueprint']['host_groups'][3]['components']), 1)
+
+ nameNodeDependencies["scope"] = "host"
+ recommendations = self.stackAdvisor.createComponentLayoutRecommendations(services, hosts)
+ # Assert that dependencies are enforced when host scope and no conditions
+ self.assertEquals(recommendations['blueprint']['host_groups'][1]['components'][1]['name'], 'NAMENODE')
+ self.assertEquals(len(recommendations['blueprint']['host_groups'][1]['components']), 2)
+
+
def test_validateRequiredComponentsPresent(self):
services = {
"Versions":