You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by rl...@apache.org on 2017/06/14 21:19:45 UTC
[14/26] ambari git commit: Revert "AMBARI-20122 - Stack advisor needs
to recommend dependency for slaves and masters"
Revert "AMBARI-20122 - Stack advisor needs to recommend dependency for slaves and masters"
This reverts commit 0b6d0dcef10ec60c83ad055567b487104573ae62.
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/75152614
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/75152614
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/75152614
Branch: refs/heads/branch-feature-AMBARI-20859
Commit: 75152614e84299d16f36a6c6d7387b77a172eb96
Parents: f1ca09c
Author: Jaimin Jetly <ja...@hortonworks.com>
Authored: Sun Jun 11 22:32:46 2017 -0700
Committer: Jaimin Jetly <ja...@hortonworks.com>
Committed: Mon Jun 12 15:46:51 2017 -0700
----------------------------------------------------------------------
.../src/main/resources/stacks/stack_advisor.py | 115 +-------------
.../stacks/2.0.6/common/test_stack_advisor.py | 153 -------------------
2 files changed, 5 insertions(+), 263 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/75152614/ambari-server/src/main/resources/stacks/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/stack_advisor.py b/ambari-server/src/main/resources/stacks/stack_advisor.py
index 6bc8fe4..4a81dc6 100644
--- a/ambari-server/src/main/resources/stacks/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/stack_advisor.py
@@ -756,19 +756,15 @@ class DefaultStackAdvisor(StackAdvisor):
if hostName not in hostsComponentsMap:
hostsComponentsMap[hostName] = []
- #Sort the services so that the dependent services will be processed before those that depend on them.
- sortedServices = self.getServicesSortedByDependencies(services)
#extend hostsComponentsMap' with MASTER components
- for service in sortedServices:
+ for service in services["services"]:
masterComponents = [component for component in service["components"] if self.isMasterComponent(component)]
serviceName = service["StackServices"]["service_name"]
serviceAdvisor = self.getServiceAdvisor(serviceName)
for component in masterComponents:
componentName = component["StackServiceComponents"]["component_name"]
advisor = serviceAdvisor if serviceAdvisor is not None else self
- #Filter the hosts such that only hosts that meet the dependencies are included (if possible)
- filteredHosts = self.getFilteredHostsBasedOnDependencies(services, component, hostsList, hostsComponentsMap)
- hostsForComponent = advisor.getHostsForMasterComponent(services, hosts, component, filteredHosts)
+ hostsForComponent = advisor.getHostsForMasterComponent(services, hosts, component, hostsList)
#extend 'hostsComponentsMap' with 'hostsForComponent'
for hostName in hostsForComponent:
@@ -782,7 +778,7 @@ class DefaultStackAdvisor(StackAdvisor):
utilizedHosts = [item for sublist in usedHostsListList for item in sublist]
freeHosts = [hostName for hostName in hostsList if hostName not in utilizedHosts]
- for service in sortedServices:
+ for service in services["services"]:
slaveClientComponents = [component for component in service["components"]
if self.isSlaveComponent(component) or self.isClientComponent(component)]
serviceName = service["StackServices"]["service_name"]
@@ -790,10 +786,7 @@ class DefaultStackAdvisor(StackAdvisor):
for component in slaveClientComponents:
componentName = component["StackServiceComponents"]["component_name"]
advisor = serviceAdvisor if serviceAdvisor is not None else self
- #Filter the hosts and free hosts such that only hosts that meet the dependencies are included (if possible)
- filteredHosts = self.getFilteredHostsBasedOnDependencies(services, component, hostsList, hostsComponentsMap)
- filteredFreeHosts = self.filterList(freeHosts, filteredHosts)
- hostsForComponent = advisor.getHostsForSlaveComponent(services, hosts, component, filteredHosts, filteredFreeHosts)
+ hostsForComponent = advisor.getHostsForSlaveComponent(services, hosts, component, hostsList, freeHosts)
#extend 'hostsComponentsMap' with 'hostsForComponent'
for hostName in hostsForComponent:
@@ -803,7 +796,7 @@ class DefaultStackAdvisor(StackAdvisor):
hostsComponentsMap[hostName].append( { "name": componentName } )
#colocate custom services
- for service in sortedServices:
+ for service in services["services"]:
serviceName = service["StackServices"]["service_name"]
serviceAdvisor = self.getServiceAdvisor(serviceName)
if serviceAdvisor is not None:
@@ -873,104 +866,6 @@ class DefaultStackAdvisor(StackAdvisor):
return hostsForComponent
- def getServicesSortedByDependencies(self, services):
- """
- Sorts the services based on their dependencies. This is limited to non-conditional host scope dependencies.
- Services with no dependencies will go first. Services with dependencies will go after the services they are dependent on.
- If there are circular dependencies, the services will go in the order in which they were processed.
- """
- processedServices = []
- sortedServices = []
-
- for service in services["services"]:
- self.sortServicesByDependencies(services, service, processedServices, sortedServices)
-
- return sortedServices
-
- def sortServicesByDependencies(self, services, service, processedServices, sortedServices):
- """
- Sorts the services based on their dependencies. This is limited to non-conditional host scope dependencies.
- Services with no dependencies will go first. Services with dependencies will go after the services they are dependent on.
- If there are circular dependencies, the services will go in the order in which they were processed.
- """
- if service in processedServices:
- return
-
- processedServices.append(service)
-
- for component in service["components"]:
- dependencies = [] if "dependencies" not in component else component['dependencies']
- for dependency in dependencies:
- # accounts only for dependencies that are not conditional
- conditionsPresent = "conditions" in dependency["Dependencies"] and dependency["Dependencies"]["conditions"]
- scope = "cluster" if "scope" not in dependency["Dependencies"] else dependency["Dependencies"]["scope"]
- if not conditionsPresent and scope == "host":
- componentName = component["StackServiceComponents"]["component_name"]
- requiredComponentName = dependency["Dependencies"]["component_name"]
- requiredService = self.getServiceForComponentName(services, requiredComponentName)
- self.sortServicesByDependencies(services, requiredService, processedServices, sortedServices)
-
- sortedServices.append(service)
-
- def getFilteredHostsBasedOnDependencies(self, services, component, hostsList, hostsComponentsMap):
- """
- Returns a list of hosts that only includes the ones which have all host scope dependencies already assigned to them.
- If an empty list would be returned, instead the full list of hosts are returned.
- In that case, we can't possibly return a valid recommended layout so we will at least return a fully filled layout.
- """
- removeHosts = []
- dependencies = [] if "dependencies" not in component else component['dependencies']
- for dependency in dependencies:
- # accounts only for dependencies that are not conditional
- conditionsPresent = "conditions" in dependency["Dependencies"] and dependency["Dependencies"]["conditions"]
- if not conditionsPresent:
- componentName = component["StackServiceComponents"]["component_name"]
- requiredComponentName = dependency["Dependencies"]["component_name"]
- requiredComponent = self.getRequiredComponent(services, requiredComponentName)
-
- # We only deal with "host" scope.
- if (requiredComponent is not None) and (requiredComponent["component_category"] != "CLIENT"):
- scope = "cluster" if "scope" not in dependency["Dependencies"] else dependency["Dependencies"]["scope"]
- if scope == "host":
- for host, hostComponents in hostsComponentsMap.iteritems():
- isRequiredIncluded = False
- for component in hostComponents:
- currentComponentName = None if "name" not in component else component["name"]
- if requiredComponentName == currentComponentName:
- isRequiredIncluded = True
- if not isRequiredIncluded:
- removeHosts.append(host)
-
- filteredHostsList = []
- for host in hostsList:
- if host not in removeHosts:
- filteredHostsList.append(host)
- return filteredHostsList
-
- def filterList(self, list, filter):
- """
- Returns the union of the two lists passed in (list and filter params).
- """
- filteredList = []
- for item in list:
- if item in filter:
- filteredList.append(item)
- return filteredList
-
- def getServiceForComponentName(self, services, componentName):
- """
- Return service for component name
-
- :type services dict
- :type componentName str
- """
- for service in services["services"]:
- for component in service["components"]:
- if self.getComponentName(component) == componentName:
- return service
-
- return None
-
def isComponentUsingCardinalityForLayout(self, componentName):
return False
http://git-wip-us.apache.org/repos/asf/ambari/blob/75152614/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
index b6f1965..41c57f6 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
@@ -191,159 +191,6 @@ class TestHDP206StackAdvisor(TestCase):
]
self.assertValidationResult(expectedItems, result)
-
- def test_handleComponentDependencies(self):
- services = {
- "Versions":
- {
- "stack_name":"HDP",
- "stack_version":"2.0.6"
- },
- "services" : [
- {
- "StackServices" : {
- "service_name" : "HDFS",
- "service_version" : "2.0.6",
- },
- "components": [
- {
- "StackServiceComponents": {
- "stack_version": "2.0.6",
- "stack_name": "HDP",
- "component_category": "MASTER",
- "is_client": False,
- "is_master": True,
- "service_name": "HDFS",
- "cardinality": "1-2",
- "hostnames": [],
- "component_name": "NAMENODE",
- "display_name": "NameNode"
- },
- "dependencies": [
- {
- "Dependencies": {
- "stack_name": "HDP",
- "stack_version": "2.0.6",
- "scope": "cluster",
- "conditions": [
- {
- "configType": "hdfs-site",
- "property": "dfs.nameservices",
- "type": "PropertyExists",
- }
- ],
- "dependent_service_name": "HDFS",
- "dependent_component_name": "NAMENODE",
- "component_name": "ZOOKEEPER_SERVER"
- }
- }
- ]
- }
- ]
- },
- {
- "StackServices" : {
- "service_name" : "ZOOKEEPER",
- "service_version" : "2.0.6",
- },
- "components": [
- {
- "StackServiceComponents": {
- "stack_version": "2.0.6",
- "stack_name": "HDP",
- "component_category": "MASTER",
- "is_client": False,
- "is_master": True,
- "service_name": "HDFS",
- "cardinality": "1-2",
- "hostnames": [],
- "component_name": "ZOOKEEPER_SERVER",
- "display_name": "ZooKeeper Server"
- },
- "dependencies": []
- }
- ]
- }
- ]
- }
-
- nameNodeDependencies = services["services"][0]["components"][0]["dependencies"][0]["Dependencies"]
-
- # Tests for master component with dependencies
-
- hosts = self.prepareHosts(["c6401.ambari.apache.org", "c6402.ambari.apache.org", "c6403.ambari.apache.org", "c6404.ambari.apache.org"])
- services["services"][1]["components"][0]["StackServiceComponents"]["hostnames"] = ["c6402.ambari.apache.org", "c6403.ambari.apache.org"]
- recommendations = self.stackAdvisor.createComponentLayoutRecommendations(services, hosts)
- # Assert that dependencies are ignored when there are conditions and cluster scope
- self.assertEquals(recommendations['blueprint']['host_groups'][3]['components'][0]['name'], 'NAMENODE')
- self.assertEquals(len(recommendations['blueprint']['host_groups'][3]['components']), 1)
-
- nameNodeDependencies["scope"] = "host"
- recommendations = self.stackAdvisor.createComponentLayoutRecommendations(services, hosts)
- # Assert that dependencies are ignored when there are conditions (even for host scope)
- self.assertEquals(recommendations['blueprint']['host_groups'][3]['components'][0]['name'], 'NAMENODE')
- self.assertEquals(len(recommendations['blueprint']['host_groups'][3]['components']), 1)
-
- nameNodeDependencies["scope"] = "cluster"
- originalConditions = nameNodeDependencies["conditions"]
- nameNodeDependencies["conditions"] = []
- recommendations = self.stackAdvisor.createComponentLayoutRecommendations(services, hosts)
- # Assert that dependencies are ignored when scope is cluster
- self.assertEquals(recommendations['blueprint']['host_groups'][3]['components'][0]['name'], 'NAMENODE')
- self.assertEquals(len(recommendations['blueprint']['host_groups'][3]['components']), 1)
-
- nameNodeDependencies["scope"] = "host"
- recommendations = self.stackAdvisor.createComponentLayoutRecommendations(services, hosts)
- # Assert that dependencies are enforced for host scope without conditions
- #self.assertEquals(recommendations, "")
- self.assertEquals(len(recommendations['blueprint']['host_groups'][0]['components']), 2)
-
- services["services"][1]["components"][0]["StackServiceComponents"]["is_master"] = False
- services["services"][1]["components"][0]["StackServiceComponents"]["component_category"] = "CLIENT"
- recommendations = self.stackAdvisor.createComponentLayoutRecommendations(services, hosts)
- # Assert that dependencies are ignored when depending on client components
- self.assertEquals(recommendations['blueprint']['host_groups'][3]['components'][0]['name'], 'NAMENODE')
- self.assertEquals(len(recommendations['blueprint']['host_groups'][3]['components']), 1)
-
- # Tests for slave component with dependencies
- services["services"][0]["components"][0]["StackServiceComponents"]["component_category"] = "SLAVE"
- services["services"][0]["components"][0]["StackServiceComponents"]["is_master"] = False
- services["services"][1]["components"][0]["StackServiceComponents"]["component_category"] = "MASTER"
- services["services"][1]["components"][0]["StackServiceComponents"]["is_master"] = True
-
- nameNodeDependencies["scope"] = "cluster"
- nameNodeDependencies["conditions"] = originalConditions
- recommendations = self.stackAdvisor.createComponentLayoutRecommendations(services, hosts)
- # Assert that dependencies are ignored when there are conditions and cluster scope
- self.assertEquals(recommendations['blueprint']['host_groups'][2]['components'][0]['name'], 'NAMENODE')
- self.assertEquals(len(recommendations['blueprint']['host_groups'][2]['components']), 1)
- self.assertEquals(recommendations['blueprint']['host_groups'][3]['components'][0]['name'], 'NAMENODE')
- self.assertEquals(len(recommendations['blueprint']['host_groups'][3]['components']), 1)
-
- nameNodeDependencies["scope"] = "host"
- recommendations = self.stackAdvisor.createComponentLayoutRecommendations(services, hosts)
- # Assert that dependencies are ignored when there are conditions (even for host scope)
- self.assertEquals(recommendations['blueprint']['host_groups'][2]['components'][0]['name'], 'NAMENODE')
- self.assertEquals(len(recommendations['blueprint']['host_groups'][2]['components']), 1)
- self.assertEquals(recommendations['blueprint']['host_groups'][3]['components'][0]['name'], 'NAMENODE')
- self.assertEquals(len(recommendations['blueprint']['host_groups'][3]['components']), 1)
-
- nameNodeDependencies["scope"] = "cluster"
- nameNodeDependencies["conditions"] = []
- recommendations = self.stackAdvisor.createComponentLayoutRecommendations(services, hosts)
- # Assert that dependencies are ignored when scope is cluster
- self.assertEquals(recommendations['blueprint']['host_groups'][2]['components'][0]['name'], 'NAMENODE')
- self.assertEquals(len(recommendations['blueprint']['host_groups'][2]['components']), 1)
- self.assertEquals(recommendations['blueprint']['host_groups'][3]['components'][0]['name'], 'NAMENODE')
- self.assertEquals(len(recommendations['blueprint']['host_groups'][3]['components']), 1)
-
- nameNodeDependencies["scope"] = "host"
- recommendations = self.stackAdvisor.createComponentLayoutRecommendations(services, hosts)
- # Assert that dependencies are enforced when host scope and no conditions
- self.assertEquals(recommendations['blueprint']['host_groups'][1]['components'][1]['name'], 'NAMENODE')
- self.assertEquals(len(recommendations['blueprint']['host_groups'][1]['components']), 2)
-
-
def test_validateRequiredComponentsPresent(self):
services = {
"Versions":