You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ja...@apache.org on 2016/01/12 00:59:51 UTC
ambari git commit: AMBARI-14494: Generate warning on deploy wizard UI
when HAWQ segment is not co-located with a datanode. (lavjain via jaoki)
Repository: ambari
Updated Branches:
refs/heads/trunk 3a9ecc092 -> 723333de6
AMBARI-14494: Generate warning on deploy wizard UI when HAWQ segment is not co-located with a datanode. (lavjain via jaoki)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/723333de
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/723333de
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/723333de
Branch: refs/heads/trunk
Commit: 723333de658ee1070d91938e419d5191f2e07d55
Parents: 3a9ecc0
Author: Jun Aoki <ja...@apache.org>
Authored: Mon Jan 11 15:59:42 2016 -0800
Committer: Jun Aoki <ja...@apache.org>
Committed: Mon Jan 11 15:59:42 2016 -0800
----------------------------------------------------------------------
.../stacks/HDP/2.3/services/stack_advisor.py | 93 +++++++++++---------
.../stacks/2.3/common/test_stack_advisor.py | 24 +++++
2 files changed, 75 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/723333de/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
index dc64e24..0da566b 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
@@ -53,50 +53,59 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
def getComponentLayoutValidations(self, services, hosts):
parentItems = super(HDP23StackAdvisor, self).getComponentLayoutValidations(services, hosts)
- hiveExists = "HIVE" in [service["StackServices"]["service_name"] for service in services["services"]]
- sparkExists = "SPARK" in [service["StackServices"]["service_name"] for service in services["services"]]
-
- if not "HAWQ" in [service["StackServices"]["service_name"] for service in services["services"]] and not sparkExists:
- return parentItems
-
+ servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+ componentsListList = [service["components"] for service in services["services"]]
+ componentsList = [item["StackServiceComponents"] for sublist in componentsListList for item in sublist]
childItems = []
- hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
- hostsCount = len(hostsList)
- componentsListList = [service["components"] for service in services["services"]]
- componentsList = [item for sublist in componentsListList for item in sublist]
- hawqMasterHosts = [hostname for component in componentsList if component["StackServiceComponents"]["component_name"] == "HAWQMASTER" for hostname in component["StackServiceComponents"]["hostnames"]]
- hawqStandbyHosts = [hostname for component in componentsList if component["StackServiceComponents"]["component_name"] == "HAWQSTANDBY" for hostname in component["StackServiceComponents"]["hostnames"]]
-
- # single node case is not analyzed because HAWQ Standby Master will not be present in single node topology due to logic in createComponentLayoutRecommendations()
- if len(hawqMasterHosts) == 1 and len(hawqStandbyHosts) == 1 and hawqMasterHosts == hawqStandbyHosts:
- message = "HAWQ Standby Master and HAWQ Master should not be deployed on the same host."
- childItems.append( { "type": 'host-component', "level": 'ERROR', "message": message, "component-name": 'HAWQSTANDBY', "host": hawqStandbyHosts[0] } )
-
- if len(hawqMasterHosts) == 1 and hostsCount > 1 and self.isLocalHost(hawqMasterHosts[0]):
- message = "HAWQ Master and Ambari Server should not be deployed on the same host. " \
- "If you leave them collocated, make sure to set HAWQ Master Port property " \
- "to a value different from the port number used by Ambari Server database."
- childItems.append( { "type": 'host-component', "level": 'WARN', "message": message, "component-name": 'HAWQMASTER', "host": hawqMasterHosts[0] } )
-
- if len(hawqStandbyHosts) == 1 and hostsCount > 1 and self.isLocalHost(hawqStandbyHosts[0]):
- message = "HAWQ Standby Master and Ambari Server should not be deployed on the same host. " \
- "If you leave them collocated, make sure to set HAWQ Master Port property " \
- "to a value different from the port number used by Ambari Server database."
- childItems.append( { "type": 'host-component', "level": 'WARN', "message": message, "component-name": 'HAWQSTANDBY', "host": hawqStandbyHosts[0] } )
-
- if "SPARK_THRIFTSERVER" in [service["StackServices"]["service_name"] for service in services["services"]]:
- if not "HIVE_SERVER" in [service["StackServices"]["service_name"] for service in services["services"]]:
- message = "SPARK_THRIFTSERVER requires HIVE services to be selected."
- childItems.append( {"type": 'host-component', "level": 'ERROR', "message": messge, "component-name": 'SPARK_THRIFTSERVER'} )
-
- hmsHosts = [component["StackServiceComponents"]["hostnames"] for component in componentsList if component["StackServiceComponents"]["component_name"] == "HIVE_METASTORE"][0] if hiveExists else []
- sparkTsHosts = [component["StackServiceComponents"]["hostnames"] for component in componentsList if component["StackServiceComponents"]["component_name"] == "SPARK_THRIFTSERVER"][0] if sparkExists else []
-
- # if Spark Thrift Server is deployed but no Hive Server is deployed
- if len(sparkTsHosts) > 0 and len(hmsHosts) == 0:
- message = "SPARK_THRIFTSERVER requires HIVE_METASTORE to be selected/deployed."
- childItems.append( { "type": 'host-component', "level": 'ERROR', "message": message, "component-name": 'SPARK_THRIFTSERVER' } )
+ if "HAWQ" in servicesList:
+ hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
+ hostsCount = len(hostsList)
+
+ hawqMasterHosts = [component["hostnames"] for component in componentsList if component["component_name"] == "HAWQMASTER"][0]
+ hawqStandbyHosts = [component["hostnames"] for component in componentsList if component["component_name"] == "HAWQSTANDBY"][0]
+ hawqSegmentHosts = [component["hostnames"] for component in componentsList if component["component_name"] == "HAWQSEGMENT"][0]
+ datanodeHosts = [component["hostnames"] for component in componentsList if component["component_name"] == "DATANODE"][0]
+
+ # Generate WARNING if any HAWQSEGMENT is not colocated with a DATANODE
+ mismatchHosts = set(hawqSegmentHosts).symmetric_difference(set(datanodeHosts))
+ if len(mismatchHosts) > 0:
+ hostsString = ', '.join(mismatchHosts)
+ message = "HAWQSEGMENT is not co-located with DATANODE on the following {0} host(s): {1}".format(len(mismatchHosts), hostsString)
+ childItems.append( { "type": 'host-component', "level": 'WARN', "message": message, "component-name": 'HAWQSEGMENT' } )
+
+ # single node case is not analyzed because HAWQ Standby Master will not be present in single node topology due to logic in createComponentLayoutRecommendations()
+ if len(hawqMasterHosts) == 1 and len(hawqStandbyHosts) == 1 and hawqMasterHosts == hawqStandbyHosts:
+ message = "HAWQ Standby Master and HAWQ Master should not be deployed on the same host."
+ childItems.append( { "type": 'host-component', "level": 'ERROR', "message": message, "component-name": 'HAWQSTANDBY', "host": hawqStandbyHosts[0] } )
+
+ if len(hawqMasterHosts) == 1 and hostsCount > 1 and self.isLocalHost(hawqMasterHosts[0]):
+ message = "HAWQ Master and Ambari Server should not be deployed on the same host. " \
+ "If you leave them collocated, make sure to set HAWQ Master Port property " \
+ "to a value different from the port number used by Ambari Server database."
+ childItems.append( { "type": 'host-component', "level": 'WARN', "message": message, "component-name": 'HAWQMASTER', "host": hawqMasterHosts[0] } )
+
+ if len(hawqStandbyHosts) == 1 and hostsCount > 1 and self.isLocalHost(hawqStandbyHosts[0]):
+ message = "HAWQ Standby Master and Ambari Server should not be deployed on the same host. " \
+ "If you leave them collocated, make sure to set HAWQ Master Port property " \
+ "to a value different from the port number used by Ambari Server database."
+ childItems.append( { "type": 'host-component', "level": 'WARN', "message": message, "component-name": 'HAWQSTANDBY', "host": hawqStandbyHosts[0] } )
+
+ if "SPARK" in servicesList:
+ if "SPARK_THRIFTSERVER" in servicesList:
+ if not "HIVE_SERVER" in servicesList:
+ message = "SPARK_THRIFTSERVER requires HIVE services to be selected."
+ childItems.append( {"type": 'host-component', "level": 'ERROR', "message": message, "component-name": 'SPARK_THRIFTSERVER'} )
+
+ hmsHosts = [component["hostnames"] for component in componentsList
+ if component["component_name"] == "HIVE_METASTORE"][0] if "HIVE" in servicesList else []
+ sparkTsHosts = [component["hostnames"] for component in componentsList
+ if component["component_name"] == "SPARK_THRIFTSERVER"][0] if "SPARK" in servicesList else []
+
+ # if Spark Thrift Server is deployed but no Hive Server is deployed
+ if len(sparkTsHosts) > 0 and len(hmsHosts) == 0:
+ message = "SPARK_THRIFTSERVER requires HIVE_METASTORE to be selected/deployed."
+ childItems.append( { "type": 'host-component', "level": 'ERROR', "message": message, "component-name": 'SPARK_THRIFTSERVER' } )
parentItems.extend(childItems)
return parentItems
http://git-wip-us.apache.org/repos/asf/ambari/blob/723333de/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
index 22fcb1b..7a8d8ce 100644
--- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
@@ -182,6 +182,30 @@ class TestHDP23StackAdvisor(TestCase):
return 'c6401.ambari.apache.org' if value is None else value
+ def test_validations_hawqsegment(self):
+ """ Test validation warning for HAWQ segment not colocated with DATANODE """
+
+ services = self.load_json("services-normal-hawq-3-hosts.json")
+ hosts = self.load_json("hosts-3-hosts.json")
+ componentsListList = [service["components"] for service in services["services"]]
+ componentsList = [item for sublist in componentsListList for item in sublist]
+
+ hawqsegmentComponent = [component["StackServiceComponents"] for component in componentsList
+ if component["StackServiceComponents"]["component_name"] == "HAWQSEGMENT"][0]
+ hawqsegmentComponent["hostnames"] = ['c6401.ambari.apache.org']
+ datanodeComponent = [component["StackServiceComponents"] for component in componentsList
+ if component["StackServiceComponents"]["component_name"] == "DATANODE"][0]
+ datanodeComponent["hostnames"] = ['c6402.ambari.apache.org']
+
+ validations = self.stackAdvisor.getComponentLayoutValidations(services, hosts)
+ expected = {'component-name': 'HAWQSEGMENT', 'message': 'HAWQSEGMENT is not co-located with DATANODE on the following 2 host(s): c6402.ambari.apache.org, c6401.ambari.apache.org', 'type': 'host-component', 'level': 'WARN'}
+ self.assertEquals(validations[0], expected)
+
+ datanodeComponent["hostnames"] = ['c6401.ambari.apache.org']
+ validations = self.stackAdvisor.getComponentLayoutValidations(services, hosts)
+ self.assertEquals(len(validations), 0)
+
+
@patch('socket.getfqdn', side_effect=fqdn_mock_result)
def test_getComponentLayoutValidations_hawq_3_Hosts(self, socket_mock):
""" Test layout validations for HAWQ components on a 3-node cluster """