You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sw...@apache.org on 2016/12/07 01:17:31 UTC
ambari git commit: AMBARI-19116. Hive2/HiveServer2 (HSI) heap size
calculation to be based on LLAP concurrency.
Repository: ambari
Updated Branches:
refs/heads/branch-feature-AMBARI-18901 548d4570a -> 512475f6f
AMBARI-19116. Hive2/HiveServer2 (HSI) heap size calculation to be based on LLAP concurrency.
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/512475f6
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/512475f6
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/512475f6
Branch: refs/heads/branch-feature-AMBARI-18901
Commit: 512475f6f42710bc85c031964a187f298308d9a6
Parents: 548d457
Author: Swapan Shridhar <ss...@hortonworks.com>
Authored: Mon Dec 5 17:35:41 2016 -0800
Committer: Swapan Shridhar <ss...@hortonworks.com>
Committed: Tue Dec 6 16:56:54 2016 -0800
----------------------------------------------------------------------
.../0.12.0.2.0/package/scripts/params_linux.py | 1 +
.../HIVE/configuration/hive-interactive-env.xml | 36 ++++++++++++++++-
.../stacks/HDP/2.5/services/stack_advisor.py | 41 ++++++++++++++++----
3 files changed, 70 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/512475f6/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
index 8df6ba7..781cf5d 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
@@ -588,6 +588,7 @@ if has_hive_interactive:
hive_interactive_env_sh_template = config['configurations']['hive-interactive-env']['content']
hive_interactive_enabled = default('/configurations/hive-interactive-env/enable_hive_interactive', False)
llap_app_java_opts = default('/configurations/hive-interactive-env/llap_java_opts', '-XX:+AlwaysPreTouch {% if java_version > 7 %}-XX:+UseG1GC -XX:TLABSize=8m -XX:+ResizeTLAB -XX:+UseNUMA -XX:+AggressiveOpts -XX:MetaspaceSize=1024m -XX:InitiatingHeapOccupancyPercent=80 -XX:MaxGCPauseMillis=200{% else %}-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseParallelGC{% endif %}')
+ hive_interactive_heapsize = config['configurations']['hive-interactive-env']['hive_heapsize']
# Service check related
if hive_transport_mode.lower() == "http":
http://git-wip-us.apache.org/repos/asf/ambari/blob/512475f6/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml
index 6e93f58..09e1741 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml
@@ -212,6 +212,40 @@
<display-name>LLAP app java opts</display-name>
<on-ambari-upgrade add="true"/>
</property>
+ <property>
+ <name>hive_heapsize</name>
+ <value>512</value>
+ <description>Hive Java heap size</description>
+ <display-name>HiveServer Interactive Heap Size</display-name>
+ <value-attributes>
+ <type>int</type>
+ <minimum>512</minimum>
+ <unit>MB</unit>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ <depends-on>
+ <property>
+ <type>hive-interactive-env</type>
+ <name>enable_hive_interactive</name>
+ </property>
+ <property>
+ <type>hive-interactive-env</type>
+ <name>num_llap_nodes</name>
+ </property>
+ <property>
+ <type>hive-interactive-site</type>
+ <name>hive.llap.daemon.queue.name</name>
+ </property>
+ <property>
+ <type>capacity-scheduler</type>
+ <name>yarn.scheduler.capacity.root.queues</name>
+ </property>
+ <property>
+ <type>hive-interactive-site</type>
+ <name>hive.server2.tez.sessions.per.default.queue</name>
+ </property>
+ </depends-on>
+ </property>
<!-- hive-env.sh -->
@@ -233,7 +267,7 @@
if [ "$SERVICE" = "metastore" ]; then
export HADOOP_HEAPSIZE={{hive_metastore_heapsize}} # Setting for HiveMetastore
else
- export HADOOP_HEAPSIZE={{hive_heapsize}} # Setting for HiveServer2 and Client
+ export HADOOP_HEAPSIZE={{hive_interactive_heapsize}} # Setting for HiveServer2 and Client
fi
export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS -Xmx${HADOOP_HEAPSIZE}m"
http://git-wip-us.apache.org/repos/asf/ambari/blob/512475f6/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
index df2faeb..edd7af1 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
@@ -820,16 +820,26 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
# Get all leaf queues.
leafQueueNames = self.getAllYarnLeafQueues(capacity_scheduler_properties)
Logger.info("YARN leaf Queues = {0}".format(leafQueueNames))
+ if len(leafQueueNames) == 0:
+ raise Fail("Queue(s) couldn't be retrieved from capacity-scheduler.")
# Check if it's 1st invocation after enabling Hive Server Interactive (config: enable_hive_interactive).
changed_configs_has_enable_hive_int = self.are_config_props_in_changed_configs(services, "hive-interactive-env",
set(['enable_hive_interactive']), False)
- llap_named_queue_selected_in_curr_invocation = False
- if changed_configs_has_enable_hive_int and services['configurations']['hive-interactive-env']['properties']['enable_hive_interactive']:
- llap_named_queue_selected_in_curr_invocation = True
- putHiveInteractiveSiteProperty('hive.llap.daemon.queue.name', llap_queue_name)
- putHiveInteractiveSiteProperty('hive.server2.tez.default.queues', llap_queue_name)
- Logger.info("'hive.llap.daemon.queue.name' and 'hive.server2.tez.default.queues' values set as : {0}".format(llap_queue_name))
+ llap_named_queue_selected_in_curr_invocation = None
+ if changed_configs_has_enable_hive_int \
+ and services['configurations']['hive-interactive-env']['properties']['enable_hive_interactive']:
+ if (len(leafQueueNames) == 1 or (len(leafQueueNames) == 2 and llap_queue_name in leafQueueNames)):
+ llap_named_queue_selected_in_curr_invocation = True
+ putHiveInteractiveSiteProperty('hive.llap.daemon.queue.name', llap_queue_name)
+ putHiveInteractiveSiteProperty('hive.server2.tez.default.queues', llap_queue_name)
+ Logger.info("'hive.llap.daemon.queue.name' and 'hive.server2.tez.default.queues' values set as : {0}".format(llap_queue_name))
+ else:
+ first_leaf_queue = list(leafQueueNames)[0] # 1st invocation, pick the 1st leaf queue and set it as selected.
+ putHiveInteractiveSiteProperty('hive.llap.daemon.queue.name', first_leaf_queue)
+ putHiveInteractiveSiteProperty('hive.server2.tez.default.queues', first_leaf_queue)
+ llap_named_queue_selected_in_curr_invocation = False
+ Logger.info("'hive.llap.daemon.queue.name' and 'hive.server2.tez.default.queues' values set as : {0}".format(first_leaf_queue))
Logger.info("llap_named_queue_selected_in_curr_invocation = {0}".format(llap_named_queue_selected_in_curr_invocation))
if (len(leafQueueNames) == 2 and (llap_daemon_selected_queue_name != None and llap_daemon_selected_queue_name == llap_queue_name) or \
@@ -1078,6 +1088,19 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
llap_xmx = max(total_mem_for_executors_per_node * 0.8, total_mem_for_executors_per_node - self.get_llap_headroom_space(services, configurations))
Logger.info("Calculated llap_app_heap_size : {0}, using following : total_mem_for_executors : {1}".format(llap_xmx, total_mem_for_executors_per_node))
+ # Calculate 'hive_heapsize' for Hive2/HiveServer2 (HSI)
+ hive_server_interactive_heapsize = None
+ hive_server_interactive_hosts = self.getHostsWithComponent("HIVE", "HIVE_SERVER_INTERACTIVE", services, hosts)
+ if hive_server_interactive_hosts is None:
+ # If its None, read the base service HDFS's DATANODE node memory, as are host are considered homogenous.
+ hive_server_interactive_hosts = self.getHostsWithComponent("HDFS", "DATANODE", services, hosts)
+ if hive_server_interactive_hosts is not None and len(hive_server_interactive_hosts) > 0:
+ host_mem = long(hive_server_interactive_hosts[0]["Hosts"]["total_mem"])
+ hive_server_interactive_heapsize = min(max(2048.0, 400.0*llap_concurrency), 3.0/8 * host_mem)
+ Logger.info("Calculated 'hive_server_interactive_heapsize' : {0}, using following : llap_concurrency : {1}, host_mem : "
+ "{2}".format(hive_server_interactive_heapsize, llap_concurrency, host_mem))
+
+
Logger.info("Updating the calculations....")
# Done with calculations, updating calculated configs.
@@ -1149,6 +1172,10 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
if cache_mem_per_node >= 64:
llap_io_enabled = 'true'
+ if hive_server_interactive_heapsize != None:
+ putHiveInteractiveEnvProperty("hive_heapsize", hive_server_interactive_heapsize)
+ Logger.info("Hive2 config 'hive_heapsize' updated. Current : {0}".format(hive_server_interactive_heapsize))
+
putHiveInteractiveSiteProperty('hive.llap.io.enabled', llap_io_enabled)
Logger.info("Hive2 config 'hive.llap.io.enabled' updated to '{0}' as part of "
"'hive.llap.io.memory.size' calculation.".format(llap_io_enabled))
@@ -1830,7 +1857,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
Logger.info("AM percent key got for '{0}' queue is : '{1}'".format(llap_daemon_selected_queue_name, llap_selected_queue_am_percent_key))
break;
if llap_selected_queue_am_percent_key is None:
- Logger.info("Returning default AM percent value : '0.1' for queue : {1}".format(llap_daemon_selected_queue_name))
+ Logger.info("Returning default AM percent value : '0.1' for queue : {0}".format(llap_daemon_selected_queue_name))
return 0.1 # Default value to use if we couldn't retrieve queue's corresponding AM Percent key.
else:
llap_selected_queue_am_percent = capacity_scheduler_properties.get(llap_selected_queue_am_percent_key)