You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sm...@apache.org on 2016/04/18 20:30:05 UTC
ambari git commit: AMBARI-15700: Stack advisor - Component layout
recommendation needs support for avoiding hosts in maintenance mode
Repository: ambari
Updated Branches:
refs/heads/trunk 58e7b1259 -> 10ae6cddf
AMBARI-15700: Stack advisor - Component layout recommendation needs support for avoiding hosts in maintenance mode
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/10ae6cdd
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/10ae6cdd
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/10ae6cdd
Branch: refs/heads/trunk
Commit: 10ae6cddfbfe27cdd0ac15297159e4da3b30910f
Parents: 58e7b12
Author: Nahappan Somasundaram <ns...@hortonworks.com>
Authored: Mon Apr 11 10:34:25 2016 -0700
Committer: Nahappan Somasundaram <ns...@hortonworks.com>
Committed: Mon Apr 18 11:29:57 2016 -0700
----------------------------------------------------------------------
.../stacks/BIGTOP/0.8/services/stack_advisor.py | 11 +--
.../stacks/HDP/2.0.6/services/stack_advisor.py | 12 +--
.../stacks/HDPWIN/2.1/services/stack_advisor.py | 11 +--
.../src/main/resources/stacks/stack_advisor.py | 34 +++++++--
.../src/test/python/TestStackAdvisor.py | 80 ++++++++++++++++++++
5 files changed, 127 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/10ae6cdd/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/stack_advisor.py
index 53591cd..701d0d4 100644
--- a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/stack_advisor.py
@@ -29,8 +29,8 @@ class BaseBIGTOP08StackAdvisor(DefaultStackAdvisor):
items = []
# Validating NAMENODE and SECONDARY_NAMENODE are on different hosts if possible
- hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
- hostsCount = len(hostsList)
+ hostsSet = set(super(BaseBIGTOP08StackAdvisor, self).getActiveHosts([host["Hosts"] for host in hosts["items"]]))
+ hostsCount = len(hostsSet)
componentsListList = [service["components"] for service in services["services"]]
componentsList = [item for sublist in componentsListList for item in sublist]
@@ -42,9 +42,10 @@ class BaseBIGTOP08StackAdvisor(DefaultStackAdvisor):
if component["StackServiceComponents"]["cardinality"] is not None:
componentName = component["StackServiceComponents"]["component_name"]
componentDisplayName = component["StackServiceComponents"]["display_name"]
- componentHostsCount = 0
+ componentHosts = []
if component["StackServiceComponents"]["hostnames"] is not None:
- componentHostsCount = len(component["StackServiceComponents"]["hostnames"])
+ componentHosts = [componentHost for componentHost in component["StackServiceComponents"]["hostnames"] if componentHost in hostsSet]
+ componentHostsCount = len(componentHosts)
cardinality = str(component["StackServiceComponents"]["cardinality"])
# cardinality types: null, 1+, 1-2, 1, ALL
message = None
@@ -71,7 +72,7 @@ class BaseBIGTOP08StackAdvisor(DefaultStackAdvisor):
# Validating host-usage
usedHostsListList = [component["StackServiceComponents"]["hostnames"] for component in componentsList if not self.isComponentNotValuable(component)]
usedHostsList = [item for sublist in usedHostsListList for item in sublist]
- nonUsedHostsList = [item for item in hostsList if item not in usedHostsList]
+ nonUsedHostsList = [item for item in hostsSet if item not in usedHostsList]
for host in nonUsedHostsList:
items.append( { "type": 'host-component', "level": 'ERROR', "message": 'Host is not used', "host": str(host) } )
http://git-wip-us.apache.org/repos/asf/ambari/blob/10ae6cdd/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index 0130483..6dd1d84 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -32,8 +32,9 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
items = []
# Validating NAMENODE and SECONDARY_NAMENODE are on different hosts if possible
- hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
- hostsCount = len(hostsList)
+ # Use a set for fast lookup
+ hostsSet = set(super(HDP206StackAdvisor, self).getActiveHosts([host["Hosts"] for host in hosts["items"]])) #[host["Hosts"]["host_name"] for host in hosts["items"]]
+ hostsCount = len(hostsSet)
componentsListList = [service["components"] for service in services["services"]]
componentsList = [item for sublist in componentsListList for item in sublist]
@@ -45,9 +46,10 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
if component["StackServiceComponents"]["cardinality"] is not None:
componentName = component["StackServiceComponents"]["component_name"]
componentDisplayName = component["StackServiceComponents"]["display_name"]
- componentHostsCount = 0
+ componentHosts = []
if component["StackServiceComponents"]["hostnames"] is not None:
- componentHostsCount = len(component["StackServiceComponents"]["hostnames"])
+ componentHosts = [componentHost for componentHost in component["StackServiceComponents"]["hostnames"] if componentHost in hostsSet]
+ componentHostsCount = len(componentHosts)
cardinality = str(component["StackServiceComponents"]["cardinality"])
# cardinality types: null, 1+, 1-2, 1, ALL
message = None
@@ -74,7 +76,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
# Validating host-usage
usedHostsListList = [component["StackServiceComponents"]["hostnames"] for component in componentsList if not self.isComponentNotValuable(component)]
usedHostsList = [item for sublist in usedHostsListList for item in sublist]
- nonUsedHostsList = [item for item in hostsList if item not in usedHostsList]
+ nonUsedHostsList = [item for item in hostsSet if item not in usedHostsList]
for host in nonUsedHostsList:
items.append( { "type": 'host-component', "level": 'ERROR', "message": 'Host is not used', "host": str(host) } )
http://git-wip-us.apache.org/repos/asf/ambari/blob/10ae6cdd/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py
index b99c484..cf52ade 100644
--- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py
@@ -31,8 +31,8 @@ class HDPWIN21StackAdvisor(DefaultStackAdvisor):
items = []
# Validating NAMENODE and SECONDARY_NAMENODE are on different hosts if possible
- hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
- hostsCount = len(hostsList)
+ hostsSet = set(super(HDPWIN21StackAdvisor, self).getActiveHosts([host["Hosts"] for host in hosts["items"]]))
+ hostsCount = len(hostsSet)
componentsListList = [service["components"] for service in services["services"]]
componentsList = [item for sublist in componentsListList for item in sublist]
@@ -44,9 +44,10 @@ class HDPWIN21StackAdvisor(DefaultStackAdvisor):
if component["StackServiceComponents"]["cardinality"] is not None:
componentName = component["StackServiceComponents"]["component_name"]
componentDisplayName = component["StackServiceComponents"]["display_name"]
- componentHostsCount = 0
+ componentHosts = []
if component["StackServiceComponents"]["hostnames"] is not None:
- componentHostsCount = len(component["StackServiceComponents"]["hostnames"])
+ componentHosts = [componentHost for componentHost in component["StackServiceComponents"]["hostnames"] if componentHost in hostsSet]
+ componentHostsCount = len(componentHosts)
cardinality = str(component["StackServiceComponents"]["cardinality"])
# cardinality types: null, 1+, 1-2, 1, ALL
message = None
@@ -73,7 +74,7 @@ class HDPWIN21StackAdvisor(DefaultStackAdvisor):
# Validating host-usage
usedHostsListList = [component["StackServiceComponents"]["hostnames"] for component in componentsList if not self.isComponentNotValuable(component)]
usedHostsList = [item for sublist in usedHostsListList for item in sublist]
- nonUsedHostsList = [item for item in hostsList if item not in usedHostsList]
+ nonUsedHostsList = [item for item in hostsSet if item not in usedHostsList]
for host in nonUsedHostsList:
items.append( { "type": 'host-component', "level": 'ERROR', "message": 'Host is not used', "host": str(host) } )
http://git-wip-us.apache.org/repos/asf/ambari/blob/10ae6cdd/ambari-server/src/main/resources/stacks/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/stack_advisor.py b/ambari-server/src/main/resources/stacks/stack_advisor.py
index 539bd25..9979e7e 100644
--- a/ambari-server/src/main/resources/stacks/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/stack_advisor.py
@@ -309,12 +309,25 @@ class DefaultStackAdvisor(StackAdvisor):
implement
"""
+ """
+ Filters the list of specified hosts object and returns
+ a list of hosts which are not in maintenance mode.
+ """
+ def getActiveHosts(self, hosts):
+ hostsList = []
+
+ if (hosts is not None):
+ hostsList = [host['host_name'] for host in hosts
+ if host.get('maintenance_state') is None or host.get('maintenance_state') == "OFF"]
+
+ return hostsList
+
def recommendComponentLayout(self, services, hosts):
"""Returns Services object with hostnames array populated for components"""
stackName = services["Versions"]["stack_name"]
stackVersion = services["Versions"]["stack_version"]
- hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
+ hostsList = self.getActiveHosts([host["Hosts"] for host in hosts["items"]])
servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
layoutRecommendations = self.createComponentLayoutRecommendations(services, hosts)
@@ -339,14 +352,17 @@ class DefaultStackAdvisor(StackAdvisor):
}
}
- hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
+ hostsList = self.getActiveHosts([host["Hosts"] for host in hosts["items"]])
+
+ # for fast lookup
+ hostsSet = set(hostsList)
hostsComponentsMap = {}
for hostName in hostsList:
if hostName not in hostsComponentsMap:
hostsComponentsMap[hostName] = []
- #extend 'hostsComponentsMap' with MASTER components
+ #extend hostsComponentsMap' with MASTER components
for service in services["services"]:
masterComponents = [component for component in service["components"] if self.isMasterComponent(component)]
for component in masterComponents:
@@ -373,7 +389,9 @@ class DefaultStackAdvisor(StackAdvisor):
#extend 'hostsComponentsMap' with 'hostsForComponent'
for hostName in hostsForComponent:
- hostsComponentsMap[hostName].append( { "name":componentName } )
+ if hostName in hostsSet:
+ hostsComponentsMap[hostName].append( { "name":componentName } )
+
#extend 'hostsComponentsMap' with Slave and Client Components
componentsListList = [service["components"] for service in services["services"]]
componentsList = [item for sublist in componentsListList for item in sublist]
@@ -422,9 +440,10 @@ class DefaultStackAdvisor(StackAdvisor):
#extend 'hostsComponentsMap' with 'hostsForComponent'
for hostName in hostsForComponent:
- if hostName not in hostsComponentsMap:
+ if hostName not in hostsComponentsMap and hostName in hostsSet:
hostsComponentsMap[hostName] = []
- hostsComponentsMap[hostName].append( { "name": componentName } )
+ if hostName in hostsSet:
+ hostsComponentsMap[hostName].append( { "name": componentName } )
#prepare 'host-group's from 'hostsComponentsMap'
host_groups = recommendations["blueprint"]["host_groups"]
@@ -621,6 +640,9 @@ class DefaultStackAdvisor(StackAdvisor):
return self.getCardinalitiesDict().get(componentName, {"min": 1, "max": 1})
def getHostForComponent(self, component, hostsList):
+ if (len(hostsList) == 0):
+ return None
+
componentName = self.getComponentName(component)
if len(hostsList) != 1:
http://git-wip-us.apache.org/repos/asf/ambari/blob/10ae6cdd/ambari-server/src/test/python/TestStackAdvisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/TestStackAdvisor.py b/ambari-server/src/test/python/TestStackAdvisor.py
index 149ae1d..87d2d15 100644
--- a/ambari-server/src/test/python/TestStackAdvisor.py
+++ b/ambari-server/src/test/python/TestStackAdvisor.py
@@ -340,6 +340,86 @@ class TestStackAdvisorInitialization(TestCase):
}
self.assertEquals(actualRecommendLayoutResponse, expectedRecommendLayoutResponse)
+ # Test with maintenance_state. One host is in maintenance mode.
+ hosts= {
+ "items": [
+ {"Hosts": {"host_name": "host1", "maintenance_state":"OFF"}},
+ {"Hosts": {"host_name": "host2", "maintenance_state":"ON"}}
+ ]
+ }
+
+ actualRecommendLayoutResponse = default_stack_advisor.recommendComponentLayout(services, hosts)
+ expectedRecommendLayoutResponse = {
+ "services": ["GANGLIA", "HBASE", "HDFS", "PIG", "TEZ", "ZOOKEEPER"],
+ "recommendations": {
+ "blueprint": {
+ "host_groups": [
+ {
+ "name": "host-group-1",
+ "components": [
+ {
+ "name": "GANGLIA_SERVER"
+ },
+ {
+ "name": "HBASE_MASTER"
+ },
+ {
+ "name": "NAMENODE"
+ },
+ {
+ "name": "SECONDARY_NAMENODE"
+ },
+ {
+ "name": "ZOOKEEPER_SERVER"
+ },
+ {
+ "name": "ZOOKEEPER_CLIENT"
+ }
+ ]
+ }
+ ]
+ },
+ "blueprint_cluster_binding":
+ {
+ "host_groups": [
+ {
+ "hosts": [{"fqdn": "host1"}],
+ "name": "host-group-1"
+ }
+ ]
+ }
+ },
+ "hosts": ["host1"],
+ "Versions": {"stack_name": "HDP1", "stack_version": "2.0.6"}
+ }
+ self.assertEquals(actualRecommendLayoutResponse, expectedRecommendLayoutResponse)
+
+ # Test with maintenance_state. Both hosts are in maintenance mode.
+ hosts= {
+ "items": [
+ {"Hosts": {"host_name": "host1", "maintenance_state":"ON"}},
+ {"Hosts": {"host_name": "host2", "maintenance_state":"ON"}}
+ ]
+ }
+
+ actualRecommendLayoutResponse = default_stack_advisor.recommendComponentLayout(services, hosts)
+
+ expectedRecommendLayoutResponse = {
+ "Versions": {"stack_name": "HDP1", "stack_version": "2.0.6"},
+ "hosts": [],
+ "services": ['GANGLIA', 'HBASE', 'HDFS', 'PIG', 'TEZ', 'ZOOKEEPER'],
+ "recommendations": {
+ "blueprint": {
+ "host_groups": []
+ },
+ "blueprint_cluster_binding": {
+ "host_groups": []
+ }
+ }
+ }
+
+ self.assertEquals(actualRecommendLayoutResponse, expectedRecommendLayoutResponse)
+
# Config groups support by default
services["config-groups"] = [{
"configurations": {