You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sm...@apache.org on 2016/12/12 06:02:14 UTC
[5/5] ambari git commit: AMBARI-18901. LLAP integration enhancements
(Swapan Sridhar via smohanty)
AMBARI-18901. LLAP integration enhancements (Swapan Sridhar via smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/ab9acef4
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/ab9acef4
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/ab9acef4
Branch: refs/heads/trunk
Commit: ab9acef421af40ddb88fe7589c16d1868fda55f3
Parents: 646cfc6
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Sun Dec 11 22:01:54 2016 -0800
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Sun Dec 11 22:01:54 2016 -0800
----------------------------------------------------------------------
.../server/configuration/Configuration.java | 1 -
.../server/upgrade/UpgradeCatalog250.java | 74 +-
.../package/scripts/hive_interactive.py | 12 +
.../package/scripts/hive_server_interactive.py | 48 +-
.../0.12.0.2.0/package/scripts/params_linux.py | 9 +
.../0.4.0.2.1/package/scripts/params_linux.py | 7 +
.../stacks/HDP/2.2/services/stack_advisor.py | 15 +-
.../stacks/HDP/2.3/services/stack_advisor.py | 2 +
.../HIVE/configuration/hive-interactive-env.xml | 80 +-
.../configuration/hive-interactive-site.xml | 68 +-
.../HIVE/configuration/tez-interactive-site.xml | 71 +-
.../HDP/2.5/services/HIVE/themes/theme.json | 27 +-
.../YARN/configuration/capacity-scheduler.xml | 2 +-
.../stacks/HDP/2.5/services/stack_advisor.py | 700 ++--
.../services/HIVE/configuration/hive-env.xml | 119 +
.../HIVE/configuration/hive-interactive-env.xml | 87 +
.../2.6/services/TEZ/configuration/tez-env.xml | 56 +
.../2.6/services/TEZ/configuration/tez-site.xml | 28 +
.../stacks/HDP/2.6/services/stack_advisor.py | 25 +-
.../server/upgrade/UpgradeCatalog250Test.java | 85 +-
.../stacks/2.2/common/test_stack_advisor.py | 7 +-
.../stacks/2.5/HIVE/test_hive_server_int.py | 2 +
.../stacks/2.5/common/test_stack_advisor.py | 3280 ++----------------
.../app/mixins/common/configs/configs_saver.js | 2 +-
24 files changed, 1458 insertions(+), 3349 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/ab9acef4/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
index 526e5be..50ec0e2 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
@@ -2531,7 +2531,6 @@ public class Configuration {
"log4j.monitor.delay", TimeUnit.MINUTES.toMillis(5));
/**
-<<<<<<< a5fdae802210ae1f8d4fed2234f1651cbe61c2b5
* Indicates whether parallel topology task creation is enabled for blueprint cluster provisioning.
* Defaults to <code>false</code>.
* @see #TOPOLOGY_TASK_PARALLEL_CREATION_THREAD_COUNT
http://git-wip-us.apache.org/repos/asf/ambari/blob/ab9acef4/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
index ff1663e..e148899 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
@@ -17,14 +17,8 @@
*/
package org.apache.ambari.server.upgrade;
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
+import com.google.inject.Inject;
+import com.google.inject.Injector;
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.actionmanager.CommandExecutionType;
import org.apache.ambari.server.controller.AmbariManagementController;
@@ -38,8 +32,13 @@ import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.inject.Inject;
-import com.google.inject.Injector;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
/**
* Upgrade catalog for version 2.5.0.
@@ -135,6 +134,8 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog {
addNewConfigurationsFromXml();
updateAMSConfigs();
updateKafkaConfigs();
+ updateHIVEInteractiveConfigs();
+ updateTEZInteractiveConfigs();
updateHiveLlapConfigs();
updateTablesForZeppelinViewRemoval();
updateAtlasConfigs();
@@ -356,5 +357,56 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog {
}
}
}
-}
+ /**
+ * Updates Hive Interactive's config in hive-interactive-site.
+ *
+ * @throws AmbariException
+ */
+ protected void updateHIVEInteractiveConfigs() throws AmbariException {
+ AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
+ Clusters clusters = ambariManagementController.getClusters();
+ if (clusters != null) {
+ Map<String, Cluster> clusterMap = clusters.getClusters();
+
+ if (clusterMap != null && !clusterMap.isEmpty()) {
+ for (final Cluster cluster : clusterMap.values()) {
+ Config hiveInteractiveSite = cluster.getDesiredConfigByType("hive-interactive-site");
+ if (hiveInteractiveSite != null) {
+ updateConfigurationProperties("hive-interactive-site", Collections.singletonMap("hive.tez.container.size",
+ "SET_ON_FIRST_INVOCATION"), true, true);
+
+ updateConfigurationProperties("hive-interactive-site", Collections.singletonMap("hive.auto.convert.join.noconditionaltask.size",
+ "1000000000"), true, true);
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Updates Tez for Hive2 Interactive's config in tez-interactive-site.
+ *
+ * @throws AmbariException
+ */
+ protected void updateTEZInteractiveConfigs() throws AmbariException {
+ AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
+ Clusters clusters = ambariManagementController.getClusters();
+ if (clusters != null) {
+ Map<String, Cluster> clusterMap = clusters.getClusters();
+
+ if (clusterMap != null && !clusterMap.isEmpty()) {
+ for (final Cluster cluster : clusterMap.values()) {
+ Config tezInteractiveSite = cluster.getDesiredConfigByType("tez-interactive-site");
+ if (tezInteractiveSite != null) {
+
+ updateConfigurationProperties("tez-interactive-site", Collections.singletonMap("tez.runtime.io.sort.mb", "512"), true, true);
+
+ updateConfigurationProperties("tez-interactive-site", Collections.singletonMap("tez.runtime.unordered.output.buffer.size-mb",
+ "100"), true, true);
+ }
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/ab9acef4/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py
index 888b920..af2a05d 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py
@@ -57,6 +57,7 @@ Sets up the configs, jdbc connection and tarball copy to HDFS for Hive Server In
@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
def hive_interactive(name=None):
import params
+ MB_TO_BYTES = 1048576
# Create Hive User Dir
params.HdfsResource(params.hive_hdfs_user_dir,
@@ -107,6 +108,17 @@ def hive_interactive(name=None):
del merged_hive_interactive_site[item]
'''
+ Config 'hive.llap.io.memory.size' calculated value in stack_advisor is in MB as of now. We need to
+ convert it to bytes before we write it down to config file.
+ '''
+ if 'hive.llap.io.memory.size' in merged_hive_interactive_site.keys():
+ hive_llap_io_mem_size_in_mb = merged_hive_interactive_site.get("hive.llap.io.memory.size")
+ hive_llap_io_mem_size_in_bytes = long(hive_llap_io_mem_size_in_mb) * MB_TO_BYTES
+ merged_hive_interactive_site['hive.llap.io.memory.size'] = hive_llap_io_mem_size_in_bytes
+ Logger.info("Converted 'hive.llap.io.memory.size' value from '{0} MB' to '{1} Bytes' before writing "
+ "it to config file.".format(hive_llap_io_mem_size_in_mb, hive_llap_io_mem_size_in_bytes))
+
+ '''
Hive2 doesn't have support for Atlas, we need to remove the Hook 'org.apache.atlas.hive.hook.HiveHook',
which would have come in config 'hive.exec.post.hooks' during the site merge logic, if Atlas is installed.
'''
http://git-wip-us.apache.org/repos/asf/ambari/blob/ab9acef4/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py
index b5edac8..e04c53c 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py
@@ -268,15 +268,29 @@ class HiveServerInteractiveDefault(HiveServerInteractive):
Logger.info("LLAP app '{0}' is not running. llap will be started.".format(LLAP_APP_NAME))
pass
+ # Call for cleaning up the earlier run(s) LLAP package folders.
+ self._cleanup_past_llap_package_dirs()
+
Logger.info("Starting LLAP")
LLAP_PACKAGE_CREATION_PATH = Script.get_tmp_dir()
unique_name = "llap-slider%s" % datetime.utcnow().strftime('%Y-%m-%d_%H-%M-%S')
+ # Figure out the Slider Anti-affinity to be used.
+ # YARN does not support anti-affinity, and therefore Slider implements AA by the means of exclusion lists, i.e, it
+ # starts containers one by one and excludes the nodes it gets (adding a delay of ~2sec./machine). When the LLAP
+ # container memory size configuration is more than half of YARN node memory, AA is implicit and should be avoided.
+ slider_placement = 4
+ if long(params.llap_daemon_container_size) > (0.5 * long(params.yarn_nm_mem)):
+ slider_placement = 0
+ Logger.info("Setting slider_placement : 0, as llap_daemon_container_size : {0} > 0.5 * "
+ "YARN NodeManager Memory({1})".format(params.llap_daemon_container_size, params.yarn_nm_mem))
+
cmd = format("{stack_root}/current/hive-server2-hive2/bin/hive --service llap --instances {params.num_llap_nodes}"
" --slider-am-container-mb {params.slider_am_container_mb} --size {params.llap_daemon_container_size}m"
" --cache {params.hive_llap_io_mem_size}m --xmx {params.llap_heap_size}m --loglevel {params.llap_log_level}"
- " {params.llap_extra_slider_opts} --output {LLAP_PACKAGE_CREATION_PATH}/{unique_name}")
+ " --slider-placement {slider_placement} --output {LLAP_PACKAGE_CREATION_PATH}/{unique_name}"
+ " {params.llap_extra_slider_opts} --skiphadoopversion --skiphbasecp --output {LLAP_PACKAGE_CREATION_PATH}/{unique_name}")
if params.security_enabled:
llap_keytab_splits = params.hive_llap_keytab_file.split("/")
Logger.debug("llap_keytab_splits : {0}".format(llap_keytab_splits))
@@ -341,6 +355,38 @@ class HiveServerInteractiveDefault(HiveServerInteractive):
raise
"""
+ Checks and deletes previous run 'LLAP package' folders, ignoring three latest packages.
+ Last three are are ignore for debugging/reference purposes.
+ Helps in keeping check on disk space used.
+ """
+ def _cleanup_past_llap_package_dirs(self):
+ try:
+ import params
+ Logger.info("Determining previous run 'LLAP package' folder(s) to be deleted ....")
+ llap_package_folder_name_prefix = "llap-slider" # Package name is like : llap-sliderYYYY-MM-DD-HH:MM:SS
+ num_folders_to_retain = 3 # Hardcoding it as of now, as no considerable use was found to provide an env param.
+ file_names = [dir_name for dir_name in os.listdir(Script.get_tmp_dir())
+ if dir_name.startswith(llap_package_folder_name_prefix)]
+
+ file_names.sort()
+ del file_names[-num_folders_to_retain:] # Ignore 'num_folders_to_retain' latest package folders.
+ Logger.info("Previous run 'LLAP package' folder(s) to be deleted = {0}".format(file_names))
+
+ if file_names:
+ for path in file_names:
+ abs_path = Script.get_tmp_dir()+"/"+path
+ if os.path.isdir(abs_path):
+ shutil.rmtree(abs_path)
+ Logger.info("Deleted previous run 'LLAP package' folder : {0}".format(abs_path))
+ else:
+ Logger.info("No '{0}*' folder deleted.".format(llap_package_folder_name_prefix))
+ except Exception as e:
+ Logger.info("Exception while doing cleanup for past 'LLAP package(s)'.")
+ traceback.print_exc()
+
+
+
+ """
Does kinit and copies keytab for Hive/LLAP to HDFS.
"""
def setup_security(self):
http://git-wip-us.apache.org/repos/asf/ambari/blob/ab9acef4/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
index f1a95ed..33f1056 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
@@ -134,6 +134,13 @@ if check_stack_feature(StackFeature.HIVE_SERVER_INTERACTIVE, version_for_stack_f
hive_interactive_bin = format('{stack_root}/current/{component_directory_interactive}/bin')
hive_interactive_lib = format('{stack_root}/current/{component_directory_interactive}/lib')
+# Heap dump related
+heap_dump_enabled = default('/configurations/hive-env/enable_heap_dump', None)
+heap_dump_opts = "" # Empty if 'heap_dump_enabled' is False.
+if heap_dump_enabled:
+ heap_dump_path = default('/configurations/hive-env/heap_dump_location', "/tmp")
+ heap_dump_opts = " -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath="+heap_dump_path
+
# Hive Interactive related paths
hive_interactive_var_lib = '/var/lib/hive2'
@@ -588,6 +595,7 @@ if has_hive_interactive:
hive_interactive_env_sh_template = config['configurations']['hive-interactive-env']['content']
hive_interactive_enabled = default('/configurations/hive-interactive-env/enable_hive_interactive', False)
llap_app_java_opts = default('/configurations/hive-interactive-env/llap_java_opts', '-XX:+AlwaysPreTouch {% if java_version > 7 %}-XX:+UseG1GC -XX:TLABSize=8m -XX:+ResizeTLAB -XX:+UseNUMA -XX:+AggressiveOpts -XX:MetaspaceSize=1024m -XX:InitiatingHeapOccupancyPercent=80 -XX:MaxGCPauseMillis=200{% else %}-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseParallelGC{% endif %}')
+ hive_interactive_heapsize = config['configurations']['hive-interactive-env']['hive_heapsize']
# Service check related
if hive_transport_mode.lower() == "http":
@@ -599,6 +607,7 @@ if has_hive_interactive:
tez_interactive_user = config['configurations']['tez-env']['tez_user']
num_retries_for_checking_llap_status = default('/configurations/hive-interactive-env/num_retries_for_checking_llap_status', 10)
# Used in LLAP slider package creation
+ yarn_nm_mem = config['configurations']['yarn-site']['yarn.nodemanager.resource.memory-mb']
num_llap_nodes = config['configurations']['hive-interactive-env']['num_llap_nodes']
llap_daemon_container_size = config['configurations']['hive-interactive-site']['hive.llap.daemon.yarn.container.mb']
llap_log_level = config['configurations']['hive-interactive-env']['llap_log_level']
http://git-wip-us.apache.org/repos/asf/ambari/blob/ab9acef4/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_linux.py
index 6587151..0322406 100644
--- a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_linux.py
@@ -63,6 +63,13 @@ if stack_version_formatted and check_stack_feature(StackFeature.CONFIG_VERSIONIN
config_path = os.path.join(stack_root, "current/tez-client/conf")
config_dir = os.path.realpath(config_path)
+# Heap dump related
+heap_dump_enabled = default('/configurations/tez-env/enable_heap_dump', None)
+heap_dump_opts = "" # Empty if 'heap_dump_enabled' is False.
+if heap_dump_enabled:
+ heap_dump_path = default('/configurations/tez-env/heap_dump_location', "/tmp")
+ heap_dump_opts = " -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath="+heap_dump_path
+
kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
security_enabled = config['configurations']['cluster-env']['security_enabled']
smokeuser = config['configurations']['cluster-env']['smokeuser']
http://git-wip-us.apache.org/repos/asf/ambari/blob/ab9acef4/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
index 100d66e..6ae10b3 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
@@ -307,6 +307,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
putWebhcatSiteProperty = self.putProperty(configurations, "webhcat-site", services)
putHiveSitePropertyAttribute = self.putPropertyAttribute(configurations, "hive-site")
putHiveEnvPropertyAttributes = self.putPropertyAttribute(configurations, "hive-env")
+ putHiveServerPropertyAttributes = self.putPropertyAttribute(configurations, "hiveserver2-site")
servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
# Storage
@@ -527,7 +528,8 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
putHiveServerProperty("hive.security.authorization.enabled", "true")
putHiveServerProperty("hive.security.authorization.manager", "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory")
putHiveServerProperty("hive.security.authenticator.manager", "org.apache.hadoop.hive.ql.security.SessionStateUserAuthenticator")
- putHiveServerProperty("hive.conf.restricted.list", "hive.security.authenticator.manager,hive.security.authorization.manager,hive.users.in.admin.role")
+ putHiveServerProperty("hive.conf.restricted.list", "hive.security.authenticator.manager,hive.security.authorization.manager,hive.security.metastore.authorization.manager,"
+ "hive.security.metastore.authenticator.manager,hive.users.in.admin.role,hive.server2.xsrf.filter.enabled,hive.security.authorization.enabled")
putHiveSiteProperty("hive.security.authorization.manager", "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdConfOnlyAuthorizerFactory")
if sqlstdauth_class not in auth_manager_values:
auth_manager_values.append(sqlstdauth_class)
@@ -543,7 +545,16 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
putHiveServerProperty("hive.security.authorization.enabled", "true")
putHiveServerProperty("hive.security.authorization.manager", "com.xasecure.authorization.hive.authorizer.XaSecureHiveAuthorizerFactory")
putHiveServerProperty("hive.security.authenticator.manager", "org.apache.hadoop.hive.ql.security.SessionStateUserAuthenticator")
- putHiveServerProperty("hive.conf.restricted.list", "hive.security.authorization.enabled,hive.security.authorization.manager,hive.security.authenticator.manager")
+ putHiveServerProperty("hive.conf.restricted.list", "hive.security.authenticator.manager,hive.security.authorization.manager,hive.security.metastore.authorization.manager,"
+ "hive.security.metastore.authenticator.manager,hive.users.in.admin.role,hive.server2.xsrf.filter.enabled,hive.security.authorization.enabled")
+
+ # hive_security_authorization == 'None'
+ if str(configurations["hive-env"]["properties"]["hive_security_authorization"]).lower() == "None":
+ putHiveSiteProperty("hive.server2.enable.doAs", "true")
+ putHiveServerProperty("hive.security.authorization.enabled", "false")
+ putHiveServerPropertyAttributes("hive.security.authorization.manager", 'delete', 'true')
+ putHiveServerPropertyAttributes("hive.security.authenticator.manager", 'delete', 'true')
+ putHiveServerPropertyAttributes("hive.conf.restricted.list", 'delete', 'true')
putHiveSiteProperty("hive.server2.use.SSL", "false")
http://git-wip-us.apache.org/repos/asf/ambari/blob/ab9acef4/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
index b6c83bd..d97efe2 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
@@ -162,6 +162,8 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
if versionSplits and len(versionSplits) > 1 and int(versionSplits[0]) > 0 and int(versionSplits[1]) > 7:
jvmGCParams = "-XX:+UseG1GC -XX:+ResizeTLAB"
putTezProperty('tez.am.launch.cmd-opts', "-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA " + jvmGCParams)
+ # Note: Same calculation is done in 2.6/stack_advisor::recommendTezConfigurations() for 'tez.task.launch.cmd-opts',
+ # and along with it, are appended heap dump opts. If something changes here, make sure to change it in 2.6 stack.
putTezProperty('tez.task.launch.cmd-opts', "-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA " + jvmGCParams)
http://git-wip-us.apache.org/repos/asf/ambari/blob/ab9acef4/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml
index 279c7c4..7b20728 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml
@@ -53,40 +53,14 @@
<on-ambari-upgrade add="true"/>
</property>
<property>
- <name>llap_queue_capacity</name>
- <value>0</value>
- <description>Percentage of the cluster dedicated to interactive query.</description>
- <display-name>% of Cluster Capacity</display-name>
- <value-attributes>
- <type>int</type>
- <minimum>20</minimum>
- <maximum>100</maximum>
- <increment-step>1</increment-step>
- </value-attributes>
- <depends-on>
- <property>
- <type>hive-interactive-env</type>
- <name>enable_hive_interactive</name>
- </property>
- <property>
- <type>hive-interactive-site</type>
- <name>hive.llap.daemon.queue.name</name>
- </property>
- <property>
- <type>capacity-scheduler</type>
- <name>yarn.scheduler.capacity.root.queues</name>
- </property>
- </depends-on>
- <on-ambari-upgrade add="true"/>
- </property>
- <property>
<name>num_llap_nodes</name>
<value>1</value>
<description>The number of Hive LLAP daemons to run.</description>
- <display-name>Number of LLAP Daemons</display-name>
+ <display-name>Number of LLAP Nodes</display-name>
<value-attributes>
<type>int</type>
<minimum>1</minimum>
+ <increment-step>1</increment-step>
</value-attributes>
<depends-on>
<property>
@@ -94,10 +68,6 @@
<name>enable_hive_interactive</name>
</property>
<property>
- <type>hive-interactive-env</type>
- <name>llap_queue_capacity</name>
- </property>
- <property>
<type>hive-interactive-site</type>
<name>hive.llap.daemon.queue.name</name>
</property>
@@ -105,10 +75,6 @@
<type>capacity-scheduler</type>
<name>yarn.scheduler.capacity.root.queues</name>
</property>
- <property>
- <type>hive-interactive-site</type>
- <name>hive.server2.tez.sessions.per.default.queue</name>
- </property>
</depends-on>
<on-ambari-upgrade add="true"/>
</property>
@@ -150,7 +116,7 @@
</property>
<property>
<type>hive-interactive-env</type>
- <name>llap_queue_capacity</name>
+ <name>num_llap_nodes</name>
</property>
<property>
<type>hive-interactive-site</type>
@@ -187,7 +153,7 @@
</property>
<property>
<type>hive-interactive-env</type>
- <name>llap_queue_capacity</name>
+ <name>num_llap_nodes</name>
</property>
<property>
<type>hive-interactive-site</type>
@@ -211,7 +177,7 @@
<display-name>LLAP's reserved headroom for YARN container</display-name>
<value-attributes>
<type>int</type>
- <unit>bytes</unit>
+ <unit>MB</unit>
</value-attributes>
<on-ambari-upgrade add="true"/>
</property>
@@ -246,6 +212,40 @@
<display-name>LLAP app java opts</display-name>
<on-ambari-upgrade add="true"/>
</property>
+ <property>
+ <name>hive_heapsize</name>
+ <value>512</value>
+ <description>Hive Java heap size</description>
+ <display-name>HiveServer Interactive Heap Size</display-name>
+ <value-attributes>
+ <type>int</type>
+ <minimum>512</minimum>
+ <unit>MB</unit>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ <depends-on>
+ <property>
+ <type>hive-interactive-env</type>
+ <name>enable_hive_interactive</name>
+ </property>
+ <property>
+ <type>hive-interactive-env</type>
+ <name>num_llap_nodes</name>
+ </property>
+ <property>
+ <type>hive-interactive-site</type>
+ <name>hive.llap.daemon.queue.name</name>
+ </property>
+ <property>
+ <type>capacity-scheduler</type>
+ <name>yarn.scheduler.capacity.root.queues</name>
+ </property>
+ <property>
+ <type>hive-interactive-site</type>
+ <name>hive.server2.tez.sessions.per.default.queue</name>
+ </property>
+ </depends-on>
+ </property>
<!-- hive-env.sh -->
@@ -267,7 +267,7 @@
if [ "$SERVICE" = "metastore" ]; then
export HADOOP_HEAPSIZE={{hive_metastore_heapsize}} # Setting for HiveMetastore
else
- export HADOOP_HEAPSIZE={{hive_heapsize}} # Setting for HiveServer2 and Client
+ export HADOOP_HEAPSIZE={{hive_interactive_heapsize}} # Setting for HiveServer2 and Client
fi
export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS -Xmx${HADOOP_HEAPSIZE}m"
http://git-wip-us.apache.org/repos/asf/ambari/blob/ab9acef4/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
index 0207e49..2fb1553 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
@@ -67,7 +67,7 @@ limitations under the License.
</property>
<property>
<type>hive-interactive-env</type>
- <name>llap_queue_capacity</name>
+ <name>num_llap_nodes</name>
</property>
<property>
<type>hive-interactive-site</type>
@@ -411,7 +411,7 @@ limitations under the License.
</property>
<property>
<type>hive-interactive-env</type>
- <name>llap_queue_capacity</name>
+ <name>num_llap_nodes</name>
</property>
<property>
<type>hive-interactive-site</type>
@@ -447,7 +447,7 @@ limitations under the License.
</property>
<property>
<type>hive-interactive-env</type>
- <name>llap_queue_capacity</name>
+ <name>num_llap_nodes</name>
</property>
<property>
<type>hive-interactive-site</type>
@@ -478,7 +478,7 @@ limitations under the License.
<name>hive.llap.daemon.yarn.container.mb</name>
<display-name>Memory per daemon</display-name>
<description>Total memory used by individual LLAP daemons. This includes memory for the cache as well as for the query execution.</description>
- <value>341</value>
+ <value>0</value>
<value-attributes>
<type>int</type>
<unit>MB</unit>
@@ -491,7 +491,7 @@ limitations under the License.
</property>
<property>
<type>hive-interactive-env</type>
- <name>llap_queue_capacity</name>
+ <name>num_llap_nodes</name>
</property>
<property>
<type>hive-interactive-site</type>
@@ -646,4 +646,62 @@ limitations under the License.
</value-attributes>
<on-ambari-upgrade add="false"/>
</property>
+ <property>
+ <name>hive.tez.container.size</name>
+ <value>SET_ON_FIRST_INVOCATION</value>
+ <depends-on>
+ <property>
+ <type>hive-interactive-env</type>
+ <name>enable_hive_interactive</name>
+ </property>
+ <property>
+ <type>hive-interactive-env</type>
+ <name>num_llap_nodes</name>
+ </property>
+ <property>
+ <type>hive-interactive-site</type>
+ <name>hive.llap.daemon.queue.name</name>
+ </property>
+ <property>
+ <type>capacity-scheduler</type>
+ <name>yarn.scheduler.capacity.root.queues</name>
+ </property>
+ <property>
+ <type>hive-interactive-site</type>
+ <name>hive.server2.tez.sessions.per.default.queue</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>hive.auto.convert.join.noconditionaltask.size</name>
+ <value>1000000000</value>
+ <description>If hive.auto.convert.join.noconditionaltask is off, this parameter does not take affect. However, if it
+ is on, and the sum of size for n-1 of the tables/partitions for a n-way join is smaller than this size, the join is directly
+ converted to a mapjoin(there is no conditional task). The default is 10MB.
+ </description>
+ <depends-on>
+ <property>
+ <type>hive-interactive-env</type>
+ <name>enable_hive_interactive</name>
+ </property>
+ <property>
+ <type>hive-interactive-env</type>
+ <name>num_llap_nodes</name>
+ </property>
+ <property>
+ <type>hive-interactive-site</type>
+ <name>hive.llap.daemon.queue.name</name>
+ </property>
+ <property>
+ <type>capacity-scheduler</type>
+ <name>yarn.scheduler.capacity.root.queues</name>
+ </property>
+ <property>
+ <type>hive-interactive-site</type>
+ <name>hive.server2.tez.sessions.per.default.queue</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="true"/>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/ab9acef4/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml
index 9e588e9..b331736 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml
@@ -83,11 +83,8 @@
</property>
<property>
<name>tez.am.resource.memory.mb</name>
- <value>1536</value>
+ <value>0</value>
<description>The amount of memory to be used by the AppMaster</description>
- <value-attributes>
- <type>int</type>
- </value-attributes>
<depends-on>
<property>
<type>hive-interactive-env</type>
@@ -95,7 +92,71 @@
</property>
<property>
<type>hive-interactive-env</type>
- <name>llap_queue_capacity</name>
+ <name>num_llap_nodes</name>
+ </property>
+ <property>
+ <type>hive-interactive-site</type>
+ <name>hive.llap.daemon.queue.name</name>
+ </property>
+ <property>
+ <type>capacity-scheduler</type>
+ <name>yarn.scheduler.capacity.root.queues</name>
+ </property>
+ <property>
+ <type>hive-interactive-site</type>
+ <name>hive.server2.tez.sessions.per.default.queue</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>tez.runtime.io.sort.mb</name>
+ <value>512</value>
+ <description>The size of the sort buffer when output needs to be sorted</description>
+ <depends-on>
+ <property>
+ <type>tez-site</type>
+ <name>tez.task.resource.memory.mb</name>
+ </property>
+ <property>
+ <type>hive-interactive-env</type>
+ <name>enable_hive_interactive</name>
+ </property>
+ <property>
+ <type>hive-interactive-env</type>
+ <name>num_llap_nodes</name>
+ </property>
+ <property>
+ <type>hive-interactive-site</type>
+ <name>hive.llap.daemon.queue.name</name>
+ </property>
+ <property>
+ <type>capacity-scheduler</type>
+ <name>yarn.scheduler.capacity.root.queues</name>
+ </property>
+ <property>
+ <type>hive-interactive-site</type>
+ <name>hive.server2.tez.sessions.per.default.queue</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>tez.runtime.unordered.output.buffer.size-mb</name>
+ <value>100</value>
+ <description>The size of the buffer when output does not require to be sorted</description>
+ <depends-on>
+ <property>
+ <type>tez-site</type>
+ <name>tez.task.resource.memory.mb</name>
+ </property>
+ <property>
+ <type>hive-interactive-env</type>
+ <name>enable_hive_interactive</name>
+ </property>
+ <property>
+ <type>hive-interactive-env</type>
+ <name>num_llap_nodes</name>
</property>
<property>
<type>hive-interactive-site</type>
http://git-wip-us.apache.org/repos/asf/ambari/blob/ab9acef4/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json
index 452537d..1d50b6a 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json
@@ -63,11 +63,7 @@
]
},
{
- "config": "hive-interactive-env/llap_queue_capacity",
- "subsection-name": "interactive-query-row1-col1"
- },
- {
- "config": "hive-interactive-site/hive.server2.tez.sessions.per.default.queue",
+ "config": "hive-interactive-env/num_llap_nodes",
"subsection-name": "interactive-query-row1-col1",
"depends-on": [
{
@@ -89,12 +85,8 @@
]
},
{
- "config": "hive-interactive-env/copy_num_llap_nodes",
+ "config": "hive-interactive-site/hive.server2.tez.sessions.per.default.queue",
"subsection-name": "interactive-query-row1-col1",
- "property_value_attributes": {
- "ui_only_property": true,
- "copy": "hive-interactive-env/num_llap_nodes"
- },
"depends-on": [
{
"configs":[
@@ -226,23 +218,12 @@
}
},
{
- "config": "hive-interactive-env/copy_num_llap_nodes",
- "widget": {
- "type": "label",
- "units": [
- {
- "unit-name": "int"
- }
- ]
- }
- },
- {
- "config": "hive-interactive-env/llap_queue_capacity",
+ "config": "hive-interactive-env/num_llap_nodes",
"widget": {
"type": "slider",
"units": [
{
- "unit-name": "percent"
+ "unit-name": "int"
}
]
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/ab9acef4/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/capacity-scheduler.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/capacity-scheduler.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/capacity-scheduler.xml
index 9ff8484..bc0ecc4 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/capacity-scheduler.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/capacity-scheduler.xml
@@ -26,7 +26,7 @@
</property>
<property>
<type>hive-interactive-env</type>
- <name>llap_queue_capacity</name>
+ <name>num_llap_nodes</name>
</property>
</depends-on>
<on-ambari-upgrade add="false"/>