You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ja...@apache.org on 2016/02/06 00:48:07 UTC
ambari git commit: AMBARI-14911: Populate hawq_site properties when
YARN is installed after HAWQ. (adenissov via jaoki)
Repository: ambari
Updated Branches:
refs/heads/trunk f57f9b291 -> 31309e28b
AMBARI-14911: Populate hawq_site properties when YARN is installed after HAWQ. (adenissov via jaoki)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/31309e28
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/31309e28
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/31309e28
Branch: refs/heads/trunk
Commit: 31309e28b470f45f4a9c44ad7e0393dd31eb2dd6
Parents: f57f9b2
Author: Jun Aoki <ja...@apache.org>
Authored: Fri Feb 5 15:47:56 2016 -0800
Committer: Jun Aoki <ja...@apache.org>
Committed: Fri Feb 5 15:47:56 2016 -0800
----------------------------------------------------------------------
.../HAWQ/2.0.0/configuration/hawq-site.xml | 12 +++++
.../stacks/HDP/2.3/services/stack_advisor.py | 37 +++++++++++---
.../stacks/2.3/common/test_stack_advisor.py | 51 +++++++++++++++++++-
3 files changed, 93 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/31309e28/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml
index f034749..f195215 100644
--- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml
+++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml
@@ -153,6 +153,12 @@
<description>
The address of YARN resource manager server.
</description>
+ <depends-on>
+ <property>
+ <type>yarn-site</type>
+ <name>yarn.resourcemanager.address</name>
+ </property>
+ </depends-on>
</property>
<property>
@@ -161,6 +167,12 @@
<description>
The address of YARN scheduler server.
</description>
+ <depends-on>
+ <property>
+ <type>yarn-site</type>
+ <name>yarn.resourcemanager.scheduler.address</name>
+ </property>
+ </depends-on>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/ambari/blob/31309e28/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
index dae3e8a..b354378 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
@@ -678,19 +678,32 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
def recommendHAWQConfigurations(self, configurations, clusterData, services, hosts):
+ if "hawq-site" not in services["configurations"]:
+ return
+ hawq_site = services["configurations"]["hawq-site"]["properties"]
putHawqSiteProperty = self.putProperty(configurations, "hawq-site", services)
- if self.isHawqMasterComponentOnAmbariServer(services):
- if "hawq-site" in services["configurations"] and "hawq_master_address_port" in services["configurations"]["hawq-site"]["properties"]:
- putHawqSiteProperty('hawq_master_address_port', '')
- # calculate optimal number of virtual segments
componentsListList = [service["components"] for service in services["services"]]
componentsList = [item["StackServiceComponents"] for sublist in componentsListList for item in sublist]
+ servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+
+ # remove master port when master is colocated with Ambari server
+ if self.isHawqMasterComponentOnAmbariServer(services) and "hawq_master_address_port" in hawq_site:
+ putHawqSiteProperty('hawq_master_address_port', '')
+
+ # calculate optimal number of virtual segments
numSegments = len(self.__getHosts(componentsList, "HAWQSEGMENT"))
# update default if segments are deployed
- if numSegments and "hawq-site" in services["configurations"] and "default_segment_num" in services["configurations"]["hawq-site"]["properties"]:
+ if numSegments and "default_segment_num" in hawq_site:
factor = 6 if numSegments < 50 else 4
putHawqSiteProperty('default_segment_num', numSegments * factor)
-
+
+ # update YARN RM urls with the values from yarn-site if YARN is installed
+ if "YARN" in servicesList and "yarn-site" in services["configurations"]:
+ yarn_site = services["configurations"]["yarn-site"]["properties"]
+ for hs_prop, ys_prop in self.getHAWQYARNPropertyMapping().items():
+ if hs_prop in hawq_site and ys_prop in yarn_site:
+ putHawqSiteProperty(hs_prop, yarn_site[ys_prop])
+
def getServiceConfigurationValidators(self):
parentValidators = super(HDP23StackAdvisor, self).getServiceConfigurationValidators()
childValidators = {
@@ -946,8 +959,20 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
display_name = 'HAWQ Segment temp directory'
self.validateIfRootDir (properties, validationItems, prop_name, display_name)
+ # 3. Check YARN RM address properties
+ servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+ if "YARN" in servicesList and "yarn-site" in configurations:
+ yarn_site = getSiteProperties(configurations, "yarn-site")
+ for hs_prop, ys_prop in self.getHAWQYARNPropertyMapping().items():
+ if hs_prop in hawq_site and ys_prop in yarn_site and hawq_site[hs_prop] != yarn_site[ys_prop]:
+ message = "Expected value: {0} (this property should have the same value as the property {1} in yarn-site)".format(yarn_site[ys_prop], ys_prop)
+ validationItems.append({"config-name": hs_prop, "item": self.getWarnItem(message)})
+
return self.toConfigurationValidationProblems(validationItems, "hawq-site")
def isComponentUsingCardinalityForLayout(self, componentName):
return componentName in ['NFS_GATEWAY', 'PHOENIX_QUERY_SERVER', 'SPARK_THRIFTSERVER']
+
+ def getHAWQYARNPropertyMapping(self):
+ return { "hawq_rm_yarn_address": "yarn.resourcemanager.address", "hawq_rm_yarn_scheduler_address": "yarn.resourcemanager.scheduler.address" }
http://git-wip-us.apache.org/repos/asf/ambari/blob/31309e28/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
index 965ce98..72e7718 100644
--- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
@@ -1323,8 +1323,14 @@ class TestHDP23StackAdvisor(TestCase):
componentsListList = [service["components"] for service in services["services"]]
componentsList = [item for sublist in componentsListList for item in sublist]
hawqSegmentComponent = [component["StackServiceComponents"] for component in componentsList if component["StackServiceComponents"]["component_name"] == "HAWQSEGMENT"][0]
- services["configurations"]["hawq-site"] = {"properties": {"default_segment_num": "24"}}
+ # setup default configuration values
+ services["configurations"]["hawq-site"] = {"properties": {"default_segment_num": "24",
+ "hawq_rm_yarn_address": "localhost:8032",
+ "hawq_rm_yarn_scheduler_address": "localhost:8030"}}
+ services["configurations"]["yarn-site"] = {"properties": {"yarn.resourcemanager.address": "host1:8050",
+ "yarn.resourcemanager.scheduler.address": "host1:8030"}}
+ services["services"].append({"StackServices" : {"service_name" : "YARN"}, "components":[]})
configurations = {}
clusterData = {}
@@ -1333,6 +1339,10 @@ class TestHDP23StackAdvisor(TestCase):
self.stackAdvisor.recommendHAWQConfigurations(configurations, clusterData, services, None)
self.assertEquals(configurations["hawq-site"]["properties"]["default_segment_num"], str(3 * 6))
+ # check derived properties
+ self.assertEquals(configurations["hawq-site"]["properties"]["hawq_rm_yarn_address"], "host1:8050")
+ self.assertEquals(configurations["hawq-site"]["properties"]["hawq_rm_yarn_scheduler_address"], "host1:8030")
+
# Test 2 - with 49 segments
hawqSegmentComponent["hostnames"] = ["host" + str(i) for i in range(49)]
self.stackAdvisor.recommendHAWQConfigurations(configurations, clusterData, services, None)
@@ -1345,6 +1355,7 @@ class TestHDP23StackAdvisor(TestCase):
# Test 4 - with no segments
configurations = {}
+ services["configurations"]["hawq-site"] = {"properties":{'hawq-site': {'properties': {}}}}
hawqSegmentComponent["hostnames"] = []
self.stackAdvisor.recommendHAWQConfigurations(configurations, clusterData, services, None)
self.assertEquals(configurations, {'hawq-site': {'properties': {}}})
@@ -1616,3 +1627,41 @@ class TestHDP23StackAdvisor(TestCase):
services["configurations"]["hdfs-site"]["properties"]["dfs.allow.truncate"] = "false"
problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations, services, hosts)
self.assertEqual(len(problems), 0)
+
+ def test_validateHAWQConfigurations(self):
+ services = self.load_json("services-hawq-3-hosts.json")
+ # setup default configuration values
+ configurations = services["configurations"]
+ configurations["hawq-site"] = {"properties": {"hawq_rm_yarn_address": "localhost:8032",
+ "hawq_rm_yarn_scheduler_address": "localhost:8030"}}
+ configurations["yarn-site"] = {"properties": {"yarn.resourcemanager.address": "host1:8050",
+ "yarn.resourcemanager.scheduler.address": "host1:8030"}}
+ services["services"].append({"StackServices" : {"service_name" : "YARN"}, "components":[]})
+ properties = configurations["hawq-site"]["properties"]
+ defaults = {}
+ hosts = {}
+
+ expected_warnings = {
+ 'hawq_rm_yarn_address': {
+ 'config-type': 'hawq-site',
+ 'message': 'Expected value: host1:8050 (this property should have the same value as the property yarn.resourcemanager.address in yarn-site)',
+ 'type': 'configuration',
+ 'config-name': 'hawq_rm_yarn_address',
+ 'level': 'WARN'
+ },
+ 'hawq_rm_yarn_scheduler_address': {
+ 'config-type': 'hawq-site',
+ 'message': 'Expected value: host1:8030 (this property should have the same value as the property yarn.resourcemanager.scheduler.address in yarn-site)',
+ 'type': 'configuration',
+ 'config-name': 'hawq_rm_yarn_scheduler_address',
+ 'level': 'WARN'
+ }
+ }
+
+ problems = self.stackAdvisor.validateHAWQConfigurations(properties, defaults, configurations, services, hosts)
+ problems_dict = {}
+ for problem in problems:
+ problems_dict[problem['config-name']] = problem
+ self.assertEqual(len(problems), 2)
+ self.assertEqual(problems_dict, expected_warnings)
+