You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by al...@apache.org on 2015/09/03 17:22:27 UTC
ambari git commit: AMBARI-12998. Add HiveServer2, Metastore,
and Client Heap Sizes to Smart Configs (alexantonenko)
Repository: ambari
Updated Branches:
refs/heads/branch-2.1 50eb30a92 -> 09c79f31b
AMBARI-12998. Add HiveServer2, Metastore, and Client Heap Sizes to Smart Configs (alexantonenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/09c79f31
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/09c79f31
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/09c79f31
Branch: refs/heads/branch-2.1
Commit: 09c79f31b34d5812ac90bb80b982f1c7af939c92
Parents: 50eb30a
Author: Alex Antonenko <hi...@gmail.com>
Authored: Thu Sep 3 18:09:28 2015 +0300
Committer: Alex Antonenko <hi...@gmail.com>
Committed: Thu Sep 3 18:22:21 2015 +0300
----------------------------------------------------------------------
.../HIVE/0.12.0.2.0/configuration/hive-env.xml | 16 ++++
.../services/HIVE/configuration/hive-env.xml | 10 ++-
.../HDP/2.2/services/HIVE/themes/theme.json | 45 ++++++++++
.../stacks/HDP/2.2/services/stack_advisor.py | 20 ++++-
.../stacks/2.2/common/test_stack_advisor.py | 87 ++++++++++++++++++++
5 files changed, 176 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/09c79f31/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml
index 784f06b..2d0a182 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml
@@ -25,12 +25,28 @@
<name>hive.client.heapsize</name>
<value>512</value>
<description>Hive Client Java heap size</description>
+ <display-name>Client Heap Size</display-name>
+ <value-attributes>
+ <type>int</type>
+ <minimum>512</minimum>
+ <maximum>2048</maximum>
+ <unit>MB</unit>
+ <increment-step>512</increment-step>
+ </value-attributes>
</property>
<property>
<name>hive.metastore.heapsize</name>
<value>1024</value>
<description>Hive Metastore Java heap size</description>
+ <display-name>Metastore Heap Size</display-name>
+ <value-attributes>
+ <type>int</type>
+ <minimum>512</minimum>
+ <maximum>2048</maximum>
+ <unit>MB</unit>
+ <increment-step>512</increment-step>
+ </value-attributes>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/ambari/blob/09c79f31/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml
index ec0381a..443648f 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml
@@ -23,8 +23,16 @@
<configuration>
<property>
<name>hive.heapsize</name>
- <value>1024</value>
+ <value>512</value>
<description>Hive Java heap size</description>
+ <display-name>HiveServer2 Heap Size</display-name>
+ <value-attributes>
+ <type>int</type>
+ <minimum>512</minimum>
+ <maximum>2048</maximum>
+ <unit>MB</unit>
+ <increment-step>512</increment-step>
+ </value-attributes>
</property>
<!-- hive-env.sh -->
http://git-wip-us.apache.org/repos/asf/ambari/blob/09c79f31/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/themes/theme.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/themes/theme.json b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/themes/theme.json
index 1a3b9fc..42bc889 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/themes/theme.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/themes/theme.json
@@ -149,6 +149,18 @@
},
{
"config": "hive-env/hive_exec_orc_storage_strategy",
+ "subsection-name": "optimization-row2-col1"
+ },
+ {
+ "config": "hive-env/hive.heapsize",
+ "subsection-name": "optimization-row2-col2"
+ },
+ {
+ "config": "hive-env/hive.metastore.heapsize",
+ "subsection-name": "optimization-row2-col2"
+ },
+ {
+ "config": "hive-env/hive.client.heapsize",
"subsection-name": "optimization-row2-col2"
},
{
@@ -250,6 +262,39 @@
}
},
{
+ "config": "hive-env/hive.heapsize",
+ "widget": {
+ "type": "slider",
+ "units": [
+ {
+ "unit-name": "MB"
+ }
+ ]
+ }
+ },
+ {
+ "config": "hive-env/hive.metastore.heapsize",
+ "widget": {
+ "type": "slider",
+ "units": [
+ {
+ "unit-name": "MB"
+ }
+ ]
+ }
+ },
+ {
+ "config": "hive-env/hive.client.heapsize",
+ "widget": {
+ "type": "slider",
+ "units": [
+ {
+ "unit-name": "MB"
+ }
+ ]
+ }
+ },
+ {
"config": "hive-site/hive.auto.convert.join.noconditionaltask.size",
"widget": {
"type": "slider",
http://git-wip-us.apache.org/repos/asf/ambari/blob/09c79f31/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
index 529df83..9a7d7a0 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
@@ -242,7 +242,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
putHiveEnvProperty = self.putProperty(configurations, "hive-env", services)
putHiveSiteProperty = self.putProperty(configurations, "hive-site", services)
putHiveSitePropertyAttribute = self.putPropertyAttribute(configurations, "hive-site")
-
+ putHiveEnvPropertyAttributes = self.putPropertyAttribute(configurations, "hive-env")
servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
# Storage
@@ -475,6 +475,24 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
("hive-site" in services["configurations"] and "hive.server2.custom.authentication.class" in services["configurations"]["hive-site"]["properties"]):
putHiveSitePropertyAttribute("hive.server2.custom.authentication.class", "delete", "true")
+ # HiveServer, Client, Metastore heapsize
+ hs_heapsize_multiplier = 3.0/8
+ hm_heapsize_multiplier = 1.0/8
+ # HiveServer2 and HiveMetastore located on the same host
+ hive_server_hosts = self.getHostsWithComponent("HIVE", "HIVE_SERVER", services, hosts)
+ hive_client_hosts = self.getHostsWithComponent("HIVE", "HIVE_CLIENT", services, hosts)
+
+ if hive_server_hosts is not None and len(hive_server_hosts):
+ hs_host_ram = hive_server_hosts[0]["Hosts"]["total_mem"]/1024
+ putHiveEnvProperty("hive.metastore.heapsize", max(512, int(hs_host_ram*hm_heapsize_multiplier)))
+ putHiveEnvProperty("hive.heapsize", max(512, int(hs_host_ram*hs_heapsize_multiplier)))
+ putHiveEnvPropertyAttributes("hive.metastore.heapsize", "maximum", max(1024, hs_host_ram))
+ putHiveEnvPropertyAttributes("hive.heapsize", "maximum", max(1024, hs_host_ram))
+
+ if hive_client_hosts is not None and len(hive_client_hosts):
+ putHiveEnvProperty("hive.client.heapsize", 1024)
+ putHiveEnvPropertyAttributes("hive.client.heapsize", "maximum", max(1024, int(hive_client_hosts[0]["Hosts"]["total_mem"]/1024)))
+
def recommendHBASEConfigurations(self, configurations, clusterData, services, hosts):
super(HDP22StackAdvisor, self).recommendHbaseConfigurations(configurations, clusterData, services, hosts)
http://git-wip-us.apache.org/repos/asf/ambari/blob/09c79f31/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
index 851c8b5..d0260f9 100644
--- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
@@ -1149,6 +1149,80 @@ class TestHDP22StackAdvisor(TestCase):
"changed-configurations": [ ]
}
+ hiveService = {
+ "services": [
+ {
+ "href": "/api/v1/stacks/HDP/versions/2.2/services/HIVE",
+ "StackServices": {
+ "service_name": "HIVE",
+ "service_version": "2.6.0.2.2",
+ "stack_name": "HDP",
+ "stack_version": "2.2"
+ },
+ "components": [
+ {
+ "StackServiceComponents": {
+ "advertise_version": "false",
+ "cardinality": "1",
+ "component_category": "MASTER",
+ "component_name": "HIVE_SERVER",
+ "display_name": "HiveServer2",
+ "is_client": "false",
+ "is_master": "true",
+ "hostnames": [
+ "c6402.ambari.apache.org"
+ ]
+ },
+ "dependencies": []
+ },
+ {
+ "StackServiceComponents": {
+ "advertise_version": "true",
+ "cardinality": "1+",
+ "component_category": "SLAVE",
+ "component_name": "HIVE_CLIENT",
+ "display_name": "Hive Client",
+ "is_client": "true",
+ "is_master": "false",
+ "hostnames": [
+ "c6402.ambari.apache.org",
+ "c6403.ambari.apache.org"
+ ]
+ },
+ "dependencies": []
+ }
+ ]
+ },
+ ],
+ "configurations": {
+ "hive-env": {
+ "properties": {
+ "hive.heapsize": "200",
+ "hive.metastore.heapsize": "200",
+ "hive.client.heapsize": "200"
+ }
+ },
+ "hive-site": {
+ "properties": {
+ "hive.server2.authentication": "none",
+ "hive.server2.authentication.ldap.url": "",
+ "hive.server2.authentication.ldap.baseDN": "",
+ "hive.server2.authentication.kerberos.keytab": "",
+ "hive.server2.authentication.kerberos.principal": "",
+ "hive.server2.authentication.pam.services": "",
+ "hive.server2.custom.authentication.class": ""
+ }
+ },
+ "hiveserver2-site": {
+ "properties": {
+ "hive.security.authorization.manager": "",
+ "hive.security.authenticator.manager": ""
+ }
+ }
+ },
+ "changed-configurations": [ ]
+ }
+
hosts = {
"items" : [
{
@@ -1286,6 +1360,19 @@ class TestHDP22StackAdvisor(TestCase):
self.assertEquals(configurations['hive-site']['property_attributes']['hive.server2.tez.default.queues'], expected['hive-site']['property_attributes']['hive.server2.tez.default.queues'])
self.assertEquals(configurations['hive-site']['properties']['hive.server2.tez.default.queues'], expected['hive-site']['properties']['hive.server2.tez.default.queues'])
+ # Hive heapsize properties
+ self.stackAdvisor.recommendHIVEConfigurations(configurations, clusterData, hiveService, hosts)
+
+ # Recommended default values
+ self.assertEquals(configurations["hive-env"]["properties"]["hive.metastore.heapsize"], "512")
+ self.assertEquals(configurations["hive-env"]["properties"]["hive.heapsize"], "703")
+ self.assertEquals(configurations["hive-env"]["properties"]["hive.client.heapsize"], "1024")
+
+ # Recommended attributes for maximum values, minimum values defined in stack definition
+ self.assertEquals(configurations["hive-env"]["property_attributes"]["hive.heapsize"]["maximum"], "1877")
+ self.assertEquals(configurations["hive-env"]["property_attributes"]["hive.metastore.heapsize"]["maximum"], "1877")
+ self.assertEquals(configurations["hive-env"]["property_attributes"]["hive.client.heapsize"]["maximum"], "1877")
+
def test_recommendMapredConfigurationAttributesWithPigService(self):
configurations = {
"mapred-site": {