You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sw...@apache.org on 2017/01/20 18:45:35 UTC
ambari git commit: AMBARI-19616. Hive Server Interactive : (1). Set
'metaspacesize' based on LLAP daemon size and (2). Fix for using
--slider-placement argument only for HDP 2.6 and onwards.
Repository: ambari
Updated Branches:
refs/heads/trunk 7e0097501 -> c46ccc51b
AMBARI-19616. Hive Server Interactive : (1). Set 'metaspacesize' based on LLAP daemon size and (2). Fix for using --slider-placement argument only for HDP 2.6 and onwards.
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c46ccc51
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c46ccc51
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c46ccc51
Branch: refs/heads/trunk
Commit: c46ccc51b3fd06acf3a830f5306a482338ef5617
Parents: 7e00975
Author: Swapan Shridhar <ss...@hortonworks.com>
Authored: Fri Jan 20 10:45:15 2017 -0800
Committer: Swapan Shridhar <ss...@hortonworks.com>
Committed: Fri Jan 20 10:45:15 2017 -0800
----------------------------------------------------------------------
.../libraries/functions/constants.py | 1 +
.../package/scripts/hive_server_interactive.py | 39 ++++++++++++--------
.../0.12.0.2.0/package/scripts/params_linux.py | 3 +-
.../HDP/2.0.6/properties/stack_features.json | 5 +++
.../HIVE/configuration/hive-interactive-env.xml | 2 +-
5 files changed, 33 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/c46ccc51/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
index 6895e34..02ce194 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
@@ -114,3 +114,4 @@ class StackFeature:
CORE_SITE_FOR_RANGER_PLUGINS_SUPPORT='core_site_for_ranger_plugins'
ATLAS_INSTALL_HOOK_PACKAGE_SUPPORT="atlas_install_hook_package_support"
ATLAS_HDFS_SITE_ON_NAMENODE_HA='atlas_hdfs_site_on_namenode_ha'
+ HIVE_INTERACTIVE_GA_SUPPORT='hive_interactive_ga'
http://git-wip-us.apache.org/repos/asf/ambari/blob/c46ccc51/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py
index d58f93a..1b6b3f2 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py
@@ -273,25 +273,27 @@ class HiveServerInteractiveDefault(HiveServerInteractive):
unique_name = "llap-slider%s" % datetime.utcnow().strftime('%Y-%m-%d_%H-%M-%S')
- # Figure out the Slider Anti-affinity to be used.
- # YARN does not support anti-affinity, and therefore Slider implements AA by the means of exclusion lists, i.e, it
- # starts containers one by one and excludes the nodes it gets (adding a delay of ~2sec./machine). When the LLAP
- # container memory size configuration is more than half of YARN node memory, AA is implicit and should be avoided.
- slider_placement = 4
- if long(params.llap_daemon_container_size) > (0.5 * long(params.yarn_nm_mem)):
- slider_placement = 0
- Logger.info("Setting slider_placement : 0, as llap_daemon_container_size : {0} > 0.5 * "
- "YARN NodeManager Memory({1})".format(params.llap_daemon_container_size, params.yarn_nm_mem))
- else:
- Logger.info("Setting slider_placement: 4, as llap_daemon_container_size : {0} <= 0.5 * "
- "YARN NodeManager Memory({1})".format(params.llap_daemon_container_size, params.yarn_nm_mem))
-
-
cmd = format("{stack_root}/current/hive-server2-hive2/bin/hive --service llap --instances {params.num_llap_nodes}"
" --slider-am-container-mb {params.slider_am_container_mb} --size {params.llap_daemon_container_size}m"
" --cache {params.hive_llap_io_mem_size}m --xmx {params.llap_heap_size}m --loglevel {params.llap_log_level}"
- " --slider-placement {slider_placement} --output {LLAP_PACKAGE_CREATION_PATH}/{unique_name}"
" {params.llap_extra_slider_opts} --skiphadoopversion --skiphbasecp --output {LLAP_PACKAGE_CREATION_PATH}/{unique_name}")
+
+ # '--slider-placement' param is supported from HDP Hive GA version.
+ if params.stack_supports_hive_interactive_ga:
+ # Figure out the Slider Anti-affinity to be used.
+ # YARN does not support anti-affinity, and therefore Slider implements AA by the means of exclusion lists, i.e, it
+ # starts containers one by one and excludes the nodes it gets (adding a delay of ~2sec./machine). When the LLAP
+ # container memory size configuration is more than half of YARN node memory, AA is implicit and should be avoided.
+ slider_placement = 4
+ if long(params.llap_daemon_container_size) > (0.5 * long(params.yarn_nm_mem)):
+ slider_placement = 0
+ Logger.info("Setting slider_placement : 0, as llap_daemon_container_size : {0} > 0.5 * "
+ "YARN NodeManager Memory({1})".format(params.llap_daemon_container_size, params.yarn_nm_mem))
+ else:
+ Logger.info("Setting slider_placement: 4, as llap_daemon_container_size : {0} <= 0.5 * "
+ "YARN NodeManager Memory({1})".format(params.llap_daemon_container_size, params.yarn_nm_mem))
+ cmd += format(" --slider-placement {slider_placement}")
+
if params.security_enabled:
llap_keytab_splits = params.hive_llap_keytab_file.split("/")
Logger.debug("llap_keytab_splits : {0}".format(llap_keytab_splits))
@@ -305,6 +307,13 @@ class HiveServerInteractiveDefault(HiveServerInteractive):
# Append args.
llap_java_args = InlineTemplate(params.llap_app_java_opts).get_content()
cmd += format(" --args \" {llap_java_args}\"")
+ # Append metaspace size to args.
+ if params.java_version > 7 and params.llap_daemon_container_size > 4096:
+ if params.llap_daemon_container_size <= 32768:
+ metaspaceSize = "256m"
+ else:
+ metaspaceSize = "1024m"
+ cmd = cmd[:-1] + " -XX:MetaspaceSize="+metaspaceSize+ "\""
run_file_path = None
try:
http://git-wip-us.apache.org/repos/asf/ambari/blob/c46ccc51/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
index d0d8d1c..e9e6337 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
@@ -92,6 +92,7 @@ stack_supports_ranger_kerberos = check_stack_feature(StackFeature.RANGER_KERBERO
stack_supports_ranger_audit_db = check_stack_feature(StackFeature.RANGER_AUDIT_DB_SUPPORT, version_for_stack_feature_checks)
stack_supports_ranger_hive_jdbc_url_change = check_stack_feature(StackFeature.RANGER_HIVE_PLUGIN_JDBC_URL, version_for_stack_feature_checks)
stack_supports_atlas_hook_for_hive_interactive = check_stack_feature(StackFeature.HIVE_INTERACTIVE_ATLAS_HOOK_REQUIRED, version_for_stack_feature_checks)
+stack_supports_hive_interactive_ga = check_stack_feature(StackFeature.HIVE_INTERACTIVE_GA_SUPPORT, version_for_stack_feature_checks)
# component ROLE directory (like hive-metastore or hive-server2-hive2)
component_directory = status_params.component_directory
@@ -608,7 +609,7 @@ if has_hive_interactive:
start_hiveserver2_interactive_path = format("{tmp_dir}/start_hiveserver2_interactive_script")
hive_interactive_env_sh_template = config['configurations']['hive-interactive-env']['content']
hive_interactive_enabled = default('/configurations/hive-interactive-env/enable_hive_interactive', False)
- llap_app_java_opts = default('/configurations/hive-interactive-env/llap_java_opts', '-XX:+AlwaysPreTouch {% if java_version > 7 %}-XX:+UseG1GC -XX:TLABSize=8m -XX:+ResizeTLAB -XX:+UseNUMA -XX:+AggressiveOpts -XX:MetaspaceSize=1024m -XX:InitiatingHeapOccupancyPercent=80 -XX:MaxGCPauseMillis=200{% else %}-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseParallelGC{% endif %}')
+ llap_app_java_opts = default('/configurations/hive-interactive-env/llap_java_opts', '-XX:+AlwaysPreTouch {% if java_version > 7 %}-XX:+UseG1GC -XX:TLABSize=8m -XX:+ResizeTLAB -XX:+UseNUMA -XX:+AggressiveOpts -XX:InitiatingHeapOccupancyPercent=80 -XX:MaxGCPauseMillis=200{% else %}-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseParallelGC{% endif %}')
hive_interactive_heapsize = hive_heapsize
llap_app_name = config['configurations']['hive-interactive-env']['llap_app_name']
# Ambari upgrade may not add this config as it will force restart of HSI (stack upgrade should)
http://git-wip-us.apache.org/repos/asf/ambari/blob/c46ccc51/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json
index 6801d5a..e4a499b 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json
@@ -372,6 +372,11 @@
"name": "atlas_hdfs_site_on_namenode_ha",
"description": "Need to create hdfs-site under atlas-conf dir when Namenode-HA is enabled.",
"min_version": "2.6.0.0"
+ },
+ {
+ "name": "hive_interactive_ga",
+ "description": "Hive Interactive GA support",
+ "min_version": "2.6.0.0"
}
]
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/c46ccc51/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-env.xml
index 6a68335..0c9ce62 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-env.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-env.xml
@@ -44,7 +44,7 @@
</property>
<property>
<name>llap_java_opts</name>
- <value>-XX:+AlwaysPreTouch {% if java_version > 7 %}-XX:+UseG1GC -XX:TLABSize=8m -XX:+ResizeTLAB -XX:+UseNUMA -XX:+AggressiveOpts -XX:MetaspaceSize=1024m -XX:InitiatingHeapOccupancyPercent=80 -XX:MaxGCPauseMillis=200{% else %}-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseParallelGC{% endif %}{{heap_dump_opts}}</value>
+ <value>-XX:+AlwaysPreTouch {% if java_version > 7 %}-XX:+UseG1GC -XX:TLABSize=8m -XX:+ResizeTLAB -XX:+UseNUMA -XX:+AggressiveOpts -XX:InitiatingHeapOccupancyPercent=80 -XX:MaxGCPauseMillis=200{% else %}-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseParallelGC{% endif %}{{heap_dump_opts}}</value>
<description>Java opts for llap application</description>
<display-name>LLAP app java opts</display-name>
<on-ambari-upgrade add="false"/>