You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by nc...@apache.org on 2016/02/01 16:42:03 UTC
[26/39] ambari git commit: AMBARI-14840: Expose default_segment_num
property for HAWQ (adenissov via jaoki)
AMBARI-14840: Expose default_segment_num property for HAWQ (adenissov via jaoki)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/69c83419
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/69c83419
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/69c83419
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 69c83419517a5d87f9c460fc2b1bb8f41e50961a
Parents: ea60d29
Author: Jun Aoki <ja...@apache.org>
Authored: Fri Jan 29 13:31:46 2016 -0800
Committer: Jun Aoki <ja...@apache.org>
Committed: Fri Jan 29 13:31:46 2016 -0800
----------------------------------------------------------------------
.../HAWQ/2.0.0/configuration/hawq-site.xml | 14 ++++++++
.../stacks/HDP/2.3/services/stack_advisor.py | 9 +++++-
.../common/services-normal-hawq-3-hosts.json | 11 +++++++
.../stacks/2.3/common/test_stack_advisor.py | 34 ++++++++++++++++++++
4 files changed, 67 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/69c83419/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml
index 90982e9..f034749 100644
--- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml
+++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml
@@ -205,4 +205,18 @@
</description>
</property>
+ <property>
+ <name>default_segment_num</name>
+ <display-name>Default Number of Virtual Segments</display-name>
+ <value>24</value>
+ <description>
+ The default number of virtual segments to use when executing a query statement. When the query is actually executed,
+ the number of virtual segments may differ from this number depending on the query's needs.
+ When expanding the cluster, you should adjust this number to reflect the number of nodes in the new cluster times the number of virtual segments per node.
+ </description>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ </property>
+
</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/69c83419/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
index 0544f5a..3e60f05 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
@@ -671,7 +671,14 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
if self.isHawqMasterComponentOnAmbariServer(services):
if "hawq-site" in services["configurations"] and "hawq_master_address_port" in services["configurations"]["hawq-site"]["properties"]:
putHawqSiteProperty('hawq_master_address_port', '')
-
+ # calculate optimal number of virtual segments
+ componentsListList = [service["components"] for service in services["services"]]
+ componentsList = [item["StackServiceComponents"] for sublist in componentsListList for item in sublist]
+ numSegments = len(self.__getHosts(componentsList, "HAWQSEGMENT"))
+ # update default if segments are deployed
+ if numSegments and "hawq-site" in services["configurations"] and "default_segment_num" in services["configurations"]["hawq-site"]["properties"]:
+ factor = 6 if numSegments < 50 else 4
+ putHawqSiteProperty('default_segment_num', numSegments * factor)
def getServiceConfigurationValidators(self):
parentValidators = super(HDP23StackAdvisor, self).getServiceConfigurationValidators()
http://git-wip-us.apache.org/repos/asf/ambari/blob/69c83419/ambari-server/src/test/python/stacks/2.3/common/services-normal-hawq-3-hosts.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/common/services-normal-hawq-3-hosts.json b/ambari-server/src/test/python/stacks/2.3/common/services-normal-hawq-3-hosts.json
index a16ab32..d10a976 100644
--- a/ambari-server/src/test/python/stacks/2.3/common/services-normal-hawq-3-hosts.json
+++ b/ambari-server/src/test/python/stacks/2.3/common/services-normal-hawq-3-hosts.json
@@ -358,6 +358,17 @@
"type" : "hawq-site.xml"
},
"dependencies" : [ ]
+ }, {
+ "href" : "/api/v1/stacks/HDP/versions/2.3/services/HAWQ/configurations/default_segment_num",
+ "StackConfigurations" : {
+ "property_depends_on" : [ ],
+ "property_name" : "default_segment_num",
+ "service_name" : "HAWQ",
+ "stack_name" : "HDP",
+ "stack_version" : "2.3",
+ "type" : "hawq-site.xml"
+ },
+ "dependencies" : [ ]
} ]
}, {
"href" : "/api/v1/stacks/HDP/versions/2.3/services/HDFS",
http://git-wip-us.apache.org/repos/asf/ambari/blob/69c83419/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
index 362a7cd..e345e2a 100644
--- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
@@ -1316,6 +1316,40 @@ class TestHDP23StackAdvisor(TestCase):
self.stackAdvisor.recommendTezConfigurations(configurations, clusterData, services, hosts)
self.assertEquals(configurations, expected)
+ def test_recommendHAWQConfigurations(self):
+
+ # original cluster data with 3 segments
+ services = self.load_json("services-normal-hawq-3-hosts.json")
+ componentsListList = [service["components"] for service in services["services"]]
+ componentsList = [item for sublist in componentsListList for item in sublist]
+ hawqSegmentComponent = [component["StackServiceComponents"] for component in componentsList if component["StackServiceComponents"]["component_name"] == "HAWQSEGMENT"][0]
+ services["configurations"]["hawq-site"] = {"properties": {"default_segment_num": "24"}}
+
+ configurations = {}
+ clusterData = {}
+
+ # Test 1 - with 3 segments
+ self.assertEquals(len(hawqSegmentComponent["hostnames"]), 3)
+ self.stackAdvisor.recommendHAWQConfigurations(configurations, clusterData, services, None)
+ self.assertEquals(configurations["hawq-site"]["properties"]["default_segment_num"], str(3 * 6))
+
+ # Test 2 - with 49 segments
+ hawqSegmentComponent["hostnames"] = ["host" + str(i) for i in range(49)]
+ self.stackAdvisor.recommendHAWQConfigurations(configurations, clusterData, services, None)
+ self.assertEquals(configurations["hawq-site"]["properties"]["default_segment_num"], str(49 * 6))
+
+ # Test 3 - with 50 segments (threshold for new factor)
+ hawqSegmentComponent["hostnames"] = ["host" + str(i) for i in range(50)]
+ self.stackAdvisor.recommendHAWQConfigurations(configurations, clusterData, services, None)
+ self.assertEquals(configurations["hawq-site"]["properties"]["default_segment_num"], str(50 * 4))
+
+ # Test 4 - with no segments
+ configurations = {}
+ hawqSegmentComponent["hostnames"] = []
+ self.stackAdvisor.recommendHAWQConfigurations(configurations, clusterData, services, None)
+ self.assertEquals(configurations, {'hawq-site': {'properties': {}}})
+
+
def test_validateHiveConfigurations(self):
properties = {"hive_security_authorization": "None",
"hive.exec.orc.default.stripe.size": "8388608",