You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sw...@apache.org on 2016/04/26 01:42:37 UTC
[2/2] ambari git commit: HiveServerInteractive. (1). AMBARI-16078.
Config calculations used by LLAP app. (2). AMBARI-16086. Hide 'llap' queue
capacity slider when 'llap' queue is not being used for LLAP app. (3).
AMBARI-16087. LLAP queue is not selected
HiveServerInteractive. (1). AMBARI-16078. Config calculations used by LLAP app. (2). AMBARI-16086. Hide 'llap' queue capacity slider when 'llap' queue is not being used for LLAP app. (3). AMBARI-16087. LLAP queue is not selected when enabling Hive Interactive Query during Install Wizard (4). AMBARI-16079. waiting for fixed number of times if the launched LLAP app is in LAUNCHING state, for it to go to RUNNING_ALL or RUNNING_PARTIAL
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/29179e76
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/29179e76
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/29179e76
Branch: refs/heads/trunk
Commit: 29179e766f957bd61feae767ec2c34655ef080b0
Parents: b9bbb6a
Author: Swapan Shridhar <ss...@hortonworks.com>
Authored: Mon Apr 25 01:18:07 2016 -0700
Committer: Swapan Shridhar <ss...@hortonworks.com>
Committed: Mon Apr 25 16:41:53 2016 -0700
----------------------------------------------------------------------
.../package/scripts/hive_server_interactive.py | 21 +-
.../0.12.0.2.0/package/scripts/params_linux.py | 19 +-
.../HIVE/configuration/hive-interactive-env.xml | 91 +
.../configuration/hive-interactive-site.xml | 103 +-
.../HDP/2.5/services/HIVE/themes/theme.json | 2 +-
.../stacks/HDP/2.5/services/stack_advisor.py | 556 +++++-
.../stacks/2.5/HIVE/test_hive_server_int.py | 6 +-
.../stacks/2.5/common/test_stack_advisor.py | 1681 +++++++++++++++++-
.../python/stacks/2.5/configs/hsi_default.json | 8 +-
9 files changed, 2340 insertions(+), 147 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/29179e76/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py
index b13189d..6bf4c0a 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py
@@ -207,9 +207,10 @@ class HiveServerInteractiveDefault(HiveServerInteractive):
unique_name = "llap-slider%s" % datetime.utcnow().strftime('%Y-%m-%d_%H-%M-%S')
- cmd = format("{stack_root}/current/hive-server2-hive2/bin/hive --service llap --instances 1 "
- "--slider-am-container-mb {slider_am_container_mb} --loglevel INFO --output {unique_name}")
-
+ cmd = format("{stack_root}/current/hive-server2-hive2/bin/hive --service llap --instances {params.num_llap_nodes}"
+ " --slider-am-container-mb {params.slider_am_container_mb} --size {params.llap_daemon_container_size}m "
+ " --cache {params.hive_llap_io_mem_size}m --xmx {params.llap_heap_size}m --loglevel {params.llap_log_level}"
+ " --output {unique_name}")
if params.security_enabled:
cmd += format(" --slider-keytab-dir .slider/keytabs/{params.hive_user}/ --slider-keytab "
"{hive_llap_keytab_file} --slider-principal {hive_headless_keytab}")
@@ -299,8 +300,11 @@ class HiveServerInteractiveDefault(HiveServerInteractive):
"""
Checks llap app status. The states can be : 'COMPLETE', 'APP_NOT_FOUND', 'RUNNING_PARTIAL', 'RUNNING_ALL' & 'LAUNCHING'.
- If app is in 'APP_NOT_FOUND', 'RUNNING_PARTIAL' and 'LAUNCHING' state, we wait for 'num_times_to_wait' to have app
- in (1). 'RUNNING_ALL' (2). 'LAUNCHING' or (3). 'RUNNING_PARTIAL' state with 80% or more 'desiredInstances' running.
+ if app is in 'APP_NOT_FOUND', 'RUNNING_PARTIAL' and 'LAUNCHING' state:
+ we wait for 'num_times_to_wait' to have app in (1). 'RUNNING_ALL' or (2). 'RUNNING_PARTIAL'
+ state with 80% or more 'desiredInstances' running and Return True
+ else :
+ Return False
Parameters: llap_app_name : deployed llap app name.
num_retries : Number of retries to check the LLAP app status.
@@ -326,7 +330,7 @@ class HiveServerInteractiveDefault(HiveServerInteractive):
Logger.error("Malformed JSON data received for LLAP app. Exiting ....")
return False
- if llap_app_info['state'].upper() in ['RUNNING_ALL', 'LAUNCHING']:
+ if llap_app_info['state'].upper() == 'RUNNING_ALL':
Logger.info(
"LLAP app '{0}' in '{1}' state.".format(llap_app_name, llap_app_info['state']))
return True
@@ -361,8 +365,9 @@ class HiveServerInteractiveDefault(HiveServerInteractive):
llap_app_info['liveInstances'],
llap_app_info['desiredInstances'],
time.time() - curr_time))
- raise Fail('App state is RUNNING_PARTIAL.')
- elif llap_app_info['state'].upper() == 'APP_NOT_FOUND':
+ raise Fail("App state is RUNNING_PARTIAL. Live Instances : '{0}', Desired Instance : '{1}'".format(llap_app_info['liveInstances'],
+ llap_app_info['desiredInstances']))
+ elif llap_app_info['state'].upper() in ['APP_NOT_FOUND', 'LAUNCHING']:
status_str = format("Slider app '{0}' current state is {1}.".format(llap_app_name, llap_app_info['state']))
Logger.info(status_str)
raise Fail(status_str)
http://git-wip-us.apache.org/repos/asf/ambari/blob/29179e76/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
index a044fda..aa561f8 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
@@ -70,7 +70,7 @@ version = default("/commandParams/version", None)
current_version = default("/hostLevelParams/current_version", None)
# When downgrading the 'version' and 'current_version' are both pointing to the downgrade-target version
-# downgrade_from_version provides the source-version the downgrade is happening from
+# downgrade_from_version provides the source-version the downgrade is happening from
downgrade_from_version = default("/commandParams/downgrade_from_version", None)
# Upgrade direction
@@ -320,8 +320,7 @@ start_metastore_script = 'startMetastore.sh'
hive_metastore_pid = status_params.hive_metastore_pid
# Hive Server Interactive
-# TODO , not being passed right now
-slider_am_container_mb = default("/configurations/hive-site-interactive/slider-am-container-mb", 341)
+slider_am_container_mb = default("/configurations/hive-interactive-env/slider_am_container_mb", 341)
hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
yarn_user = config['configurations']['yarn-env']['yarn_user']
@@ -340,7 +339,7 @@ start_metastore_path = format("{tmp_dir}/start_metastore_script")
hadoop_heapsize = config['configurations']['hadoop-env']['hadoop_heapsize']
if 'role' in config and config['role'] in ["HIVE_SERVER", "HIVE_METASTORE"]:
- if stack_version_formatted_major and check_stack_feature(StackFeature.HIVE_ENV_HEAPSIZE, stack_version_formatted_major):
+ if stack_version_formatted_major and check_stack_feature(StackFeature.HIVE_ENV_HEAPSIZE, stack_version_formatted_major):
hive_heapsize = config['configurations']['hive-env']['hive.heapsize']
else:
hive_heapsize = config['configurations']['hive-site']['hive.heapsize']
@@ -418,7 +417,7 @@ if OSCheck.is_ubuntu_family():
mysql_configname = '/etc/mysql/my.cnf'
else:
mysql_configname = '/etc/my.cnf'
-
+
mysql_user = 'mysql'
# Hive security
@@ -516,6 +515,12 @@ if has_hive_interactive:
tez_interactive_config_dir = "/etc/tez_hive2/conf"
tez_interactive_user = config['configurations']['tez-env']['tez_user']
num_retries_for_checking_llap_status = default('/configurations/hive-interactive-env/num_retries_for_checking_llap_status', 10)
+ # Used in LLAP slider package creation
+ num_llap_nodes = config['configurations']['hive-interactive-env']['num_llap_nodes']
+ llap_daemon_container_size = config['configurations']['hive-interactive-site']['hive.llap.daemon.yarn.container.mb']
+ llap_log_level = config['configurations']['hive-interactive-env']['llap_log_level']
+ hive_llap_io_mem_size = config['configurations']['hive-interactive-site']['hive.llap.io.memory.size']
+ llap_heap_size = config['configurations']['hive-interactive-env']['llap_heap_size']
if security_enabled:
hive_llap_keytab_file = config['configurations']['hive-interactive-site']['hive.llap.zk.sm.keytab.file']
hive_headless_keytab = config['configurations']['hive-interactive-site']['hive.llap.zk.sm.principal']
@@ -576,7 +581,7 @@ if has_ranger_admin:
ranger_jdbc_jar_name = default("/hostLevelParams/custom_sqlanywhere_jdbc_name", None)
audit_jdbc_url = format('jdbc:sqlanywhere:database={xa_audit_db_name};host={xa_db_host}')
jdbc_driver = "sap.jdbc4.sqlanywhere.IDriver"
-
+
ranger_downloaded_custom_connector = format("{tmp_dir}/{ranger_jdbc_jar_name}") if stack_supports_ranger_audit_db else None
ranger_driver_curl_source = format("{jdk_location}/{ranger_jdbc_jar_name}") if stack_supports_ranger_audit_db else None
ranger_driver_curl_target = format("{hive_lib}/{ranger_jdbc_jar_name}") if stack_supports_ranger_audit_db else None
@@ -589,7 +594,7 @@ if has_ranger_admin:
'jdbc.url': format("{hive_url}/default;principal={hive_principal}") if security_enabled else hive_url,
'commonNameForCertificate': common_name_for_certificate
}
-
+
hive_ranger_plugin_repo = {
'isActive': 'true',
'config': json.dumps(hive_ranger_plugin_config),
http://git-wip-us.apache.org/repos/asf/ambari/blob/29179e76/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml
index 816e207..aad9c47 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml
@@ -67,6 +67,18 @@
<type>hive-interactive-env</type>
<name>enable_hive_interactive</name>
</property>
+ <property>
+ <type>hive-interactive-env</type>
+ <name>num_llap_nodes</name>
+ </property>
+ <property>
+ <type>hive-interactive-site</type>
+ <name>hive.llap.daemon.queue.name</name>
+ </property>
+ <property>
+ <type>capacity-scheduler</type>
+ <name>yarn.scheduler.capacity.root.queues</name>
+ </property>
</depends-on>
</property>
<property>
@@ -80,6 +92,24 @@
<maximum>8</maximum>
<increment-step>1</increment-step>
</value-attributes>
+ <depends-on>
+ <property>
+ <type>hive-interactive-env</type>
+ <name>enable_hive_interactive</name>
+ </property>
+ <property>
+ <type>hive-interactive-env</type>
+ <name>llap_queue_capacity</name>
+ </property>
+ <property>
+ <type>hive-interactive-site</type>
+ <name>hive.llap.daemon.queue.name</name>
+ </property>
+ <property>
+ <type>capacity-scheduler</type>
+ <name>yarn.scheduler.capacity.root.queues</name>
+ </property>
+ </depends-on>
</property>
<property>
<name>num_retries_for_checking_llap_status</name>
@@ -93,6 +123,67 @@
<increment-step>1</increment-step>
</value-attributes>
</property>
+ <property>
+ <name>llap_heap_size</name>
+ <value>0</value>
+ <description>Heap Size used by LLAP app.</description>
+ <display-name>LLAP heap size</display-name>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <depends-on>
+ <property>
+ <type>hive-interactive-env</type>
+ <name>enable_hive_interactive</name>
+ </property>
+ <property>
+ <type>hive-interactive-env</type>
+ <name>llap_queue_capacity</name>
+ </property>
+ <property>
+ <type>hive-interactive-site</type>
+ <name>hive.llap.daemon.queue.name</name>
+ </property>
+ <property>
+ <type>capacity-scheduler</type>
+ <name>yarn.scheduler.capacity.root.queues</name>
+ </property>
+ </depends-on>
+ </property>
+ <property>
+ <name>slider_am_container_mb</name>
+ <value>341</value>
+ <description>Slider's app master container size in MB.</description>
+ <display-name>Slider AM container size</display-name>
+ <value-attributes>
+ <type>int</type>
+ <unit>MB</unit>
+ </value-attributes>
+ <depends-on>
+ <property>
+ <type>hive-interactive-env</type>
+ <name>enable_hive_interactive</name>
+ </property>
+ <property>
+ <type>hive-interactive-env</type>
+ <name>llap_queue_capacity</name>
+ </property>
+ <property>
+ <type>hive-interactive-site</type>
+ <name>hive.llap.daemon.queue.name</name>
+ </property>
+ <property>
+ <type>capacity-scheduler</type>
+ <name>yarn.scheduler.capacity.root.queues</name>
+ </property>
+ </depends-on>
+ </property>
+ <property>
+ <name>llap_log_level</name>
+ <value>INFO</value>
+ <description>LLAP app logging level</description>
+ <display-name>LLAP app logging level</display-name>
+ </property>
<!-- hive-env.sh -->
<property>
http://git-wip-us.apache.org/repos/asf/ambari/blob/29179e76/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
index 21d1109..bd65435 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
@@ -51,6 +51,28 @@ limitations under the License.
<maximum>10</maximum>
<increment-step>1</increment-step>
</value-attributes>
+ <depends-on>
+ <property>
+ <type>hive-interactive-env</type>
+ <name>enable_hive_interactive</name>
+ </property>
+ <property>
+ <type>hive-interactive-env</type>
+ <name>llap_queue_capacity</name>
+ </property>
+ <property>
+ <type>hive-interactive-env</type>
+ <name>num_llap_nodes</name>
+ </property>
+ <property>
+ <type>hive-interactive-site</type>
+ <name>hive.llap.daemon.queue.name</name>
+ </property>
+ <property>
+ <type>capacity-scheduler</type>
+ <name>yarn.scheduler.capacity.root.queues</name>
+ </property>
+ </depends-on>
</property>
<property>
@@ -172,21 +194,6 @@ limitations under the License.
</property>
<property>
- <name>hive.server2.tez.sessions.per.default.queue</name>
- <value>1</value>
- <description>
- The maximum number of queries this Hive LLAP cluster will be able to handle concurrently.
- </description>
- <display-name>Maximum Concurrent Queries</display-name>
- <value-attributes>
- <type>int</type>
- <minimum>1</minimum>
- <maximum>10</maximum>
- <increment-step>1</increment-step>
- </value-attributes>
- </property>
-
- <property>
<name>hive.server2.tez.default.queues</name>
<display-name>Default query queues</display-name>
<value>default</value>
@@ -378,6 +385,28 @@ limitations under the License.
<increment-step>256</increment-step>
<overridable>false</overridable>
</value-attributes>
+ <depends-on>
+ <property>
+ <type>hive-interactive-env</type>
+ <name>enable_hive_interactive</name>
+ </property>
+ <property>
+ <type>hive-interactive-env</type>
+ <name>llap_queue_capacity</name>
+ </property>
+ <property>
+ <type>hive-interactive-env</type>
+ <name>num_llap_nodes</name>
+ </property>
+ <property>
+ <type>hive-interactive-site</type>
+ <name>hive.llap.daemon.queue.name</name>
+ </property>
+ <property>
+ <type>capacity-scheduler</type>
+ <name>yarn.scheduler.capacity.root.queues</name>
+ </property>
+ </depends-on>
</property>
<property>
@@ -391,6 +420,28 @@ limitations under the License.
<maximum>8</maximum>
<increment-step>1</increment-step>
</value-attributes>
+ <depends-on>
+ <property>
+ <type>hive-interactive-env</type>
+ <name>enable_hive_interactive</name>
+ </property>
+ <property>
+ <type>hive-interactive-env</type>
+ <name>llap_queue_capacity</name>
+ </property>
+ <property>
+ <type>hive-interactive-env</type>
+ <name>num_llap_nodes</name>
+ </property>
+ <property>
+ <type>hive-interactive-site</type>
+ <name>hive.llap.daemon.queue.name</name>
+ </property>
+ <property>
+ <type>capacity-scheduler</type>
+ <name>yarn.scheduler.capacity.root.queues</name>
+ </property>
+ </depends-on>
</property>
<property>
@@ -416,6 +467,28 @@ limitations under the License.
<increment-step>256</increment-step>
<overridable>false</overridable>
</value-attributes>
+ <depends-on>
+ <property>
+ <type>hive-interactive-env</type>
+ <name>enable_hive_interactive</name>
+ </property>
+ <property>
+ <type>hive-interactive-env</type>
+ <name>llap_queue_capacity</name>
+ </property>
+ <property>
+ <type>hive-interactive-env</type>
+ <name>num_llap_nodes</name>
+ </property>
+ <property>
+ <type>hive-interactive-site</type>
+ <name>hive.llap.daemon.queue.name</name>
+ </property>
+ <property>
+ <type>capacity-scheduler</type>
+ <name>yarn.scheduler.capacity.root.queues</name>
+ </property>
+ </depends-on>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/ambari/blob/29179e76/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json
index 4e1a54b..818b970 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json
@@ -70,7 +70,7 @@
"configs":[
"hive-interactive-env/enable_hive_interactive"
],
- "if": "${hive-interactive-env/enable_hive_interactive}",
+ "if": "${hive-interactive-env/enable_hive_interactive} && ${hive-interactive-site/hive.llap.daemon.queue.name} === llap",
"then": {
"property_value_attributes": {
"visible": true
http://git-wip-us.apache.org/repos/asf/ambari/blob/29179e76/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
index 7f63d0c..ae6cede 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
@@ -17,13 +17,18 @@ See the License for the specific language governing permissions and
limitations under the License.
"""
+import math
+import traceback
+
from resource_management.core.logger import Logger
+from resource_management.core.exceptions import Fail
class HDP25StackAdvisor(HDP24StackAdvisor):
def __init__(self):
super(HDP25StackAdvisor, self).__init__()
Logger.initialize_logger()
+ self.HIVE_INTERACTIVE_SITE = 'hive-interactive-site'
def createComponentLayoutRecommendations(self, services, hosts):
parentComponentLayoutRecommendations = super(HDP25StackAdvisor, self).createComponentLayoutRecommendations(
@@ -143,15 +148,15 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
putHiveInteractiveEnvProperty('enable_hive_interactive', 'true')
putHiveInteractiveEnvProperty('hive_server_interactive_host', hsi_host)
- if 'hive.llap.zk.sm.connectionString' in services['configurations']['hive-interactive-site']['properties']:
+ if 'hive.llap.zk.sm.connectionString' in services['configurations'][self.HIVE_INTERACTIVE_SITE]['properties']:
# Fill the property 'hive.llap.zk.sm.connectionString' required by Hive Server Interactive (HiveServer2)
zookeeper_host_port = self.getZKHostPortString(services)
if zookeeper_host_port:
- putHiveInteractiveSiteProperty = self.putProperty(configurations, "hive-interactive-site", services)
+ putHiveInteractiveSiteProperty = self.putProperty(configurations, self.HIVE_INTERACTIVE_SITE, services)
putHiveInteractiveSiteProperty("hive.llap.zk.sm.connectionString", zookeeper_host_port)
# Update 'hive.llap.daemon.queue.name' if capacity scheduler is changed.
- if 'hive.llap.daemon.queue.name' in services['configurations']['hive-interactive-site']['properties']:
+ if 'hive.llap.daemon.queue.name' in services['configurations'][self.HIVE_INTERACTIVE_SITE]['properties']:
self.setLlapDaemonQueueName(services, configurations)
else:
putHiveInteractiveEnvProperty('enable_hive_interactive', 'false')
@@ -163,18 +168,493 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
super(HDP25StackAdvisor, self).recommendYARNConfigurations(configurations, clusterData, services, hosts)
# Queue 'llap' creation/removal logic (Used by Hive Interactive server and associated LLAP)
- hsi_hosts = self.__getHostsForComponent(services, "HIVE", "HIVE_SERVER_INTERACTIVE")
- if len(hsi_hosts) > 0:
- if 'hive-interactive-env' in services['configurations'] and \
- 'enable_hive_interactive' in services['configurations']['hive-interactive-env']['properties']:
- enable_hive_interactive = services['configurations']['hive-interactive-env']['properties']['enable_hive_interactive']
- llap_queue_name = 'llap'
+ if 'hive-interactive-env' in services['configurations'] and \
+ 'enable_hive_interactive' in services['configurations']['hive-interactive-env']['properties']:
+ enable_hive_interactive = services['configurations']['hive-interactive-env']['properties']['enable_hive_interactive']
+ LLAP_QUEUE_NAME = 'llap'
+
+ # Hive Server interactive is already added or getting added
+ if enable_hive_interactive == 'true':
+ self.checkAndManageLlapQueue(services, configurations, LLAP_QUEUE_NAME)
+ self.updateLlapConfigs(configurations, services, hosts, LLAP_QUEUE_NAME)
+ else: # When Hive Interactive Server is in 'off/removed' state.
+ self.checkAndStopLlapQueue(services, configurations, LLAP_QUEUE_NAME)
+
+ """
+ Entry point for updating Hive's 'LLAP app' configs namely : (1). num_llap_nodes (2). hive.llap.daemon.yarn.container.mb
+ (3). hive.llap.daemon.num.executors (4). hive.llap.io.memory.size (5). llap_heap_size (6). slider_am_container_mb.
+
+ The trigger point for updating LLAP configs (mentioned above) is change in values of :
+ 'llap_queue_capacity' or 'num_llap_nodes' or 'llap' named queue get selected for config 'hive.llap.daemon.queue.name'.
+
+ 'llap_queue_capacity', 'hive.llap.daemon.queue.name' : Change detection for this property triggers change for all
+ the configs mentioned.
+
+ 'num_llap_nodes' : If there is a change in value for 'num_llap_nodes', it's value is not updated/calulated,
+ but other dependent configs get calculated.
+ """
+ def updateLlapConfigs(self, configurations, services, hosts, llap_queue_name):
+ putHiveInteractiveSiteProperty = self.putProperty(configurations, self.HIVE_INTERACTIVE_SITE, services)
+ putHiveInteractiveSitePropertyAttribute = self.putPropertyAttribute(configurations, self.HIVE_INTERACTIVE_SITE)
+
+ putHiveInteractiveEnvProperty = self.putProperty(configurations, "hive-interactive-env", services)
+ putHiveInteractiveEnvPropertyAttribute = self.putPropertyAttribute(configurations, "hive-interactive-env")
+
+ num_llap_nodes_in_changed_configs = False
+ llap_daemon_selected_queue_name = None
+ llap_queue_selected_in_current_call = None
+
+ try:
+ if self.HIVE_INTERACTIVE_SITE in services['configurations'] and \
+ 'hive.llap.daemon.queue.name' in services['configurations'][self.HIVE_INTERACTIVE_SITE]['properties']:
+ llap_daemon_selected_queue_name = services['configurations'][self.HIVE_INTERACTIVE_SITE]['properties']['hive.llap.daemon.queue.name']
+
+ if 'hive.llap.daemon.queue.name' in configurations[self.HIVE_INTERACTIVE_SITE]['properties']:
+ llap_queue_selected_in_current_call = configurations[self.HIVE_INTERACTIVE_SITE]['properties']['hive.llap.daemon.queue.name']
+
+ # Update Visibility of LLAP configs.
+ capacitySchedulerProperties = self.getCapacitySchedulerProperties(services)
+ if capacitySchedulerProperties:
+ # Get all leaf queues.
+ leafQueueNames = self.getAllYarnLeafQueues(capacitySchedulerProperties)
+ if (llap_daemon_selected_queue_name != None and llap_daemon_selected_queue_name == llap_queue_name) or \
+ (llap_queue_selected_in_current_call != None and llap_queue_selected_in_current_call == llap_queue_name):
+ putHiveInteractiveEnvPropertyAttribute("llap_queue_capacity", "visible", "true")
+ Logger.debug("Selected YARN queue is '{0}'. Setting LLAP queue capacity slider visibility to True".format(llap_queue_name))
+ else:
+ putHiveInteractiveEnvPropertyAttribute("llap_queue_capacity", "visible", "false")
+ Logger.debug("Queue selected for LLAP app is : '{0}'. Current YARN queues : {1}. "
+ "Setting LLAP queue capacity slider visibility to False. "
+ "Skipping updating values for LLAP related configs".format(llap_daemon_selected_queue_name, list(leafQueueNames)))
+ return
+ else:
+ Logger.error("Couldn't retrieve 'capacity-scheduler' properties while doing YARN queue adjustment for Hive Server Interactive."
+ " Not calculating LLAP configs.")
+ return
+
+ # Won't be calculating values if queues not equal to 2 and queue in use is not llap
+ if len(leafQueueNames) !=2:
+ return
+
+ # 'llap' queue exists at this point.
+ if 'changed-configurations' in services.keys():
+ llap_queue_prop_old_val = None
- # Hive Server interactive is already added or getting added
- if enable_hive_interactive == 'true':
- self.checkAndManageLlapQueue(services, configurations, llap_queue_name)
- else: # When Hive Interactive Server is in 'off/removed' state.
- self.checkAndStopLlapQueue(services, configurations, llap_queue_name)
+ # Calculations are triggered only if there is change in following props : 'llap_queue_capacity' or 'enable_hive_interactive',
+ # 'num_llap_nodes' or hive.llap.daemon.queue.name has change in value and selection is 'llap'
+ config_names_to_be_checked = set(['llap_queue_capacity', 'enable_hive_interactive', 'num_llap_nodes'])
+ changed_configs_in_hive_int_env = self.are_config_props_in_changed_configs(services, "hive-interactive-env",
+ config_names_to_be_checked, False)
+
+ llap_queue_prop_in_changed_configs = self.are_config_props_in_changed_configs(services, self.HIVE_INTERACTIVE_SITE,
+ set(['hive.llap.daemon.queue.name']), False)
+
+ if not changed_configs_in_hive_int_env and not llap_queue_prop_in_changed_configs:
+ Logger.info("LLAP parameters not modified. Not adjusting LLAP configs. "
+ "Current changed-configuration received is : {0}".format(services["changed-configurations"]))
+ return
+
+ node_manager_hosts = self.get_node_manager_hosts(services, hosts)
+ node_manager_host_cnt = len(node_manager_hosts)
+
+ # If changed configurations contains 'num_llap_nodes' prop, we don't calulate it and use the same value.
+ num_llap_nodes_in_changed_configs = self.are_config_props_in_changed_configs(services, "hive-interactive-env",
+ set(['num_llap_nodes']), False)
+
+ # Get value for prop 'num_llap_nodes'.
+ if not num_llap_nodes_in_changed_configs:
+ num_llap_nodes, num_llap_nodes_max_limit = self.calculate_num_llap_nodes(services, hosts, configurations)
+ else:
+ num_llap_nodes = self.get_num_llap_nodes(services)
+ num_llap_nodes_max_limit = node_manager_host_cnt
+
+ # Get calculated value for prop 'hive.llap.daemon.yarn.container.mb'
+ llap_container_size, llap_container_size_max_limit = self.calculate_llap_app_container_size(services, hosts, configurations)
+
+ # Get calculated value for prop 'hive.llap.daemon.num.executors'
+ num_executors_per_node, num_executors_per_node_max_limit = self.calculate_llap_daemon_executors_count(services,
+ llap_container_size)
+ assert (num_executors_per_node >= 0), "'Number of executors per node' : {0}. Expected value : > 0".format(
+ num_executors_per_node)
+
+ # Get calculated value for prop 'hive.llap.io.memory.size'
+ cache_size_per_node, cache_size_per_node_max_limit = self.calculate_llap_cache_size_per_executor(services,
+ llap_container_size,
+ num_executors_per_node)
+ assert (cache_size_per_node >= 0), "'Cache size per node' : {0}. Expected value : > 0".format(
+ cache_size_per_node)
+
+ # Get calculated value for prop 'llap_heap_size'
+ llap_xmx = self.calculate_llap_app_heap_size(services, num_executors_per_node)
+
+ # Get calculated Slider AM container Size
+ yarn_min_container_size = self.get_yarn_min_container_size(services)
+ slider_am_container_mb = self.calculate_slider_am_size(yarn_min_container_size)
+
+ num_llap_nodes = long(num_llap_nodes)
+ num_llap_nodes_max_limit= long(num_llap_nodes_max_limit)
+ llap_container_size = long(llap_container_size)
+ llap_container_size_max_limit = long(llap_container_size_max_limit)
+ num_executors_per_node = long(num_executors_per_node)
+ num_executors_per_node_max_limit = long(num_executors_per_node_max_limit)
+ cache_size_per_node = long(cache_size_per_node)
+ cache_size_per_node_max_limit = long(cache_size_per_node_max_limit)
+ llap_xmx = long(llap_xmx)
+ slider_am_container_mb = long(slider_am_container_mb)
+
+ Logger.info("Setting following value for 'hive.llap.daemon.yarn.container.mb'. Min : {0}, Curr: {1}, Max: {2}"\
+ .format(llap_container_size, llap_container_size, llap_container_size_max_limit))
+ Logger.info("Setting following value for 'hive.llap.daemon.num.executors'. Min : {0}, Curr: {1}, Max: {2}"\
+ .format('1', num_executors_per_node, num_executors_per_node_max_limit))
+ Logger.info("Setting following value for 'hive.llap.io.memory.size'. Min : {0}, Curr: {1}, Max: {2}"\
+ .format('0', cache_size_per_node, cache_size_per_node_max_limit))
+ Logger.info("Setting following value for 'llap_heap_size'. Curr: {0}".format(llap_xmx))
+ Logger.info("Setting following value for 'slider_am_container_mb'. Curr: {0}".format(slider_am_container_mb))
+
+ if not num_llap_nodes_in_changed_configs:
+ Logger.info("Setting following value for 'num_llap_nodes'. Min : {0}, Curr: {1}, Max: {2}" \
+ .format('1', num_llap_nodes, num_llap_nodes_max_limit))
+ putHiveInteractiveEnvProperty('num_llap_nodes', num_llap_nodes)
+ putHiveInteractiveEnvPropertyAttribute('num_llap_nodes', "minimum", 1)
+ putHiveInteractiveEnvPropertyAttribute('num_llap_nodes', "maximum", int(num_llap_nodes_max_limit))
+
+ putHiveInteractiveSiteProperty('hive.llap.daemon.yarn.container.mb', llap_container_size)
+ putHiveInteractiveSitePropertyAttribute('hive.llap.daemon.yarn.container.mb', "minimum",
+ llap_container_size)
+ putHiveInteractiveSitePropertyAttribute('hive.llap.daemon.yarn.container.mb', "maximum",
+ llap_container_size_max_limit)
+
+ putHiveInteractiveSiteProperty('hive.llap.daemon.num.executors', num_executors_per_node)
+ putHiveInteractiveSitePropertyAttribute('hive.llap.daemon.num.executors', "minimum", 1)
+ putHiveInteractiveSitePropertyAttribute('hive.llap.daemon.num.executors', "maximum",
+ num_executors_per_node_max_limit)
+
+ putHiveInteractiveSiteProperty('hive.llap.io.memory.size', cache_size_per_node)
+ putHiveInteractiveSitePropertyAttribute('hive.llap.io.memory.size', "minimum", 0) # 0 -> Disables caching.
+ putHiveInteractiveSitePropertyAttribute('hive.llap.io.memory.size', "maximum",
+ cache_size_per_node_max_limit)
+
+ putHiveInteractiveEnvProperty('llap_heap_size', llap_xmx)
+
+ putHiveInteractiveEnvProperty('slider_am_container_mb', slider_am_container_mb)
+
+
+ except Exception as e:
+ Logger.info(e.message+" Skipping calculating LLAP configs.")
+ Logger.info(traceback.print_exc())
+
+ """
+ Checks for the presence of passed-in configuration properties in a given config, if they are changed.
+ Reads from services["changed-configurations"].
+ Parameters:
+ services: Configuration information for the cluster
+ config_type : Type of the configuration
+ config_names_set : Set of configuration properties to be checked if they are changed.
+ all_exists: If True : returns True only if all properties mentioned in 'config_names_set' we found
+ in services["changed-configurations"].
+ Otherwise, returns False.
+ If False : return True, if any of the properties mentioned in config_names_set we found in
+ services["changed-configurations"].
+ Otherwise, returns False.
+ """
+ def are_config_props_in_changed_configs(self, services, config_type, config_names_set, all_exists):
+ changedConfigs = services["changed-configurations"]
+ changed_config_names_set = set()
+ for changedConfig in changedConfigs:
+ if changedConfig['type'] == config_type:
+ changed_config_names_set.update([changedConfig['name']])
+
+ if changed_config_names_set is None:
+ return False
+ else:
+ configs_intersection = changed_config_names_set.intersection(config_names_set)
+ if all_exists:
+ if configs_intersection == config_names_set:
+ return True
+ else:
+ if len(configs_intersection) > 0 :
+ return True
+ return False
+
+ """
+ Returns all the Node Manager configs in cluster.
+ """
+ def get_node_manager_hosts(self, services, hosts):
+ if len(hosts["items"]) > 0:
+ node_manager_hosts = self.getHostsWithComponent("YARN", "NODEMANAGER", services, hosts)
+ assert (node_manager_hosts is not None), "Information about NODEMANAGER not found in cluster."
+ return node_manager_hosts
+
+
+ """
+ Returns the current LLAP queue capacity percentage value. (llap_queue_capacity)
+ """
+ def get_llap_cap_percent_slider(self, services, configurations):
+ if 'llap_queue_capacity' in services['configurations']['hive-interactive-env']['properties']:
+ llap_slider_cap_percentage = float(
+ services['configurations']['hive-interactive-env']['properties']['llap_queue_capacity'])
+ if llap_slider_cap_percentage <= 0 :
+ if 'hive-interactive-env' in configurations and \
+ 'llap_queue_capacity' in configurations["hive-interactive-env"]["properties"]:
+ llap_slider_cap_percentage = configurations["hive-interactive-env"]["properties"]["llap_queue_capacity"]
+ assert (llap_slider_cap_percentage > 0), "'llap_queue_capacity' is set to 0."
+ return llap_slider_cap_percentage
+
+
+ """
+ Returns current value of number of LLAP nodes in cluster (num_llap_nodes)
+ """
+ def get_num_llap_nodes(self, services):
+ if 'num_llap_nodes' in services['configurations']['hive-interactive-env']['properties']:
+ num_llap_nodes = float(
+ services['configurations']['hive-interactive-env']['properties']['num_llap_nodes'])
+ assert (num_llap_nodes > 0), "Number of LLAP nodes read : {0}. Expected value : > 0".format(
+ num_llap_nodes)
+ return num_llap_nodes
+ else:
+ raise Fail("Couldn't retrieve Hive Server interactive's 'num_llap_nodes' config.")
+
+ """
+ Calculates recommended and maximum LLAP nodes in the cluster.
+ """
+ def calculate_num_llap_nodes(self, services, hosts, configurations):
+ # TODO : Read NodeManager confis and figure the smallest sized NM.
+ size_of_smallest_nm = self.get_yarn_rm_mem_in_mb(services)
+ assert (
+ size_of_smallest_nm > 0), "Size of smallest NODEMANAGER calculated value : {0}. Expected value : > 0".format(
+ size_of_smallest_nm)
+ yarn_min_container_size = self.get_yarn_min_container_size(services)
+ node_size_usable = self._normalizeDown(size_of_smallest_nm, yarn_min_container_size)
+ cap_available_for_daemons = self.calculate_cap_available_for_llap_daemons(services, hosts, configurations)
+ num_llap_nodes = float(math.ceil(cap_available_for_daemons / node_size_usable))
+ assert (num_llap_nodes > 0), "Number of LLAP nodes calculated : {0}. Expected value : > 0".format(
+ num_llap_nodes)
+ # Maximum number of nodes that LLAP can use.
+ num_llap_nodes_max_limit = len(self.get_node_manager_hosts(services, hosts))
+ Logger.info("Calculated num_llap_nodes {3}, num_llap_nodes_max_limit : {4}, using following : "
+ "yarn_min_container_size : {0}, node_size_usable : {1}, cap_available_for_daemons :"
+ " {2}. ".format(yarn_min_container_size, node_size_usable, \
+ cap_available_for_daemons, num_llap_nodes, num_llap_nodes_max_limit))
+ return num_llap_nodes, num_llap_nodes_max_limit
+
+
+ """
+ Gets Tez container size (hive.tez.container.size)
+ """
+ def get_tez_container_size(self, services):
+ hive_container_size = 0
+ if 'hive.tez.container.size' in services['configurations'][self.HIVE_INTERACTIVE_SITE]['properties']:
+ hive_container_size = float(
+ services['configurations'][self.HIVE_INTERACTIVE_SITE]['properties']['hive.tez.container.size'])
+ assert (
+ hive_container_size > 0), "'hive.tez.container.size' current value : {0}. Expected value : > 0".format(
+ hive_container_size)
+ else:
+ raise Fail("Couldn't retrieve Hive Server interactive's 'hive.tez.container.size' config.")
+ return hive_container_size
+
+
+
+ """
+ Gets YARN's mimimum container size (yarn.scheduler.minimum-allocation-mb)
+ """
+ def get_yarn_min_container_size(self, services):
+ yarn_min_container_size = 0
+ if 'yarn.scheduler.minimum-allocation-mb' in services['configurations']['yarn-site']['properties']:
+ yarn_min_container_size = float(
+ services['configurations']['yarn-site']['properties']['yarn.scheduler.minimum-allocation-mb'])
+ assert (
+ yarn_min_container_size > 0), "'yarn.scheduler.minimum-allocation-mb' current value : {0}. Expected value : > 0".format(
+ yarn_min_container_size)
+ else:
+ raise Fail("Couldn't retrieve YARN's 'yarn.scheduler.minimum-allocation-mb' config.")
+ return yarn_min_container_size
+
+ """
+ Calculates recommended and maximum container size for LLAP app.
+ """
+ def calculate_llap_app_container_size(self, services, hosts, configurations):
+ cap_available_for_daemons = self.calculate_cap_available_for_llap_daemons(services, hosts, configurations)
+
+ node_manager_hosts = self.get_node_manager_hosts(services, hosts)
+ node_manager_host_cnt = len(node_manager_hosts)
+
+ num_llap_nodes_in_changed_configs = self.are_config_props_in_changed_configs(services, "hive-interactive-env", "num_llap_nodes", False)
+ if not num_llap_nodes_in_changed_configs:
+ num_llap_nodes, num_llap_nodes_max_limit = self.calculate_num_llap_nodes(services, hosts, configurations)
+ else:
+ num_llap_nodes = self.get_num_llap_nodes(services)
+
+ llap_container_size_raw = cap_available_for_daemons / num_llap_nodes
+ llap_container_size_raw_max_limit = cap_available_for_daemons / node_manager_host_cnt
+
+ yarn_min_container_size = self.get_yarn_min_container_size(services)
+
+ llap_container_size = self._normalizeDown(llap_container_size_raw, yarn_min_container_size)
+ llap_container_size_max_limit = self._normalizeDown(llap_container_size_raw_max_limit, yarn_min_container_size)
+ if llap_container_size_max_limit < llap_container_size:
+ llap_container_size_max_limit = llap_container_size
+ Logger.info("Calculated llap_container_size : {0}, llap_container_size_max_limit : {1}, using following : "
+ "cap_available_for_daemons : {2}, node_manager_host_cnt : {3}, llap_container_size_raw : {4}, "
+ "llap_container_size_raw_max_limit : {5}, yarn_min_container_size : {6} "\
+ .format(llap_container_size, llap_container_size_max_limit, cap_available_for_daemons, node_manager_host_cnt,
+ llap_container_size_raw, llap_container_size_raw_max_limit, yarn_min_container_size))
+ return llap_container_size, llap_container_size_max_limit
+
+
+ def calculate_cap_available_for_llap_daemons(self, services, hosts, configurations):
+ llap_concurrency = 0
+ llap_slider_cap_percentage = self.get_llap_cap_percent_slider(services, configurations)
+ yarn_rm_mem_in_mb = self.get_yarn_rm_mem_in_mb(services)
+
+
+ node_manager_hosts = self.get_node_manager_hosts(services, hosts)
+ assert (node_manager_hosts is not None), "Information about NODEMANAGER not found in cluster."
+
+ total_cluster_cap = len(node_manager_hosts) * yarn_rm_mem_in_mb
+
+ total_llap_queue_size = float(llap_slider_cap_percentage) / 100 * total_cluster_cap
+
+ llap_daemon_container_size = self.get_hive_am_container_size(services)
+
+ yarn_min_container_size = self.get_yarn_min_container_size(services)
+
+ if 'hive.server2.tez.sessions.per.default.queue' in services['configurations'][self.HIVE_INTERACTIVE_SITE][
+ 'properties']:
+ llap_concurrency = float(services['configurations'][self.HIVE_INTERACTIVE_SITE]['properties'][
+ 'hive.server2.tez.sessions.per.default.queue'])
+ assert (llap_concurrency > 0), "'hive.server2.tez.sessions.per.default.queue' current value : {0}. Expected value : > 0"\
+ .format(llap_concurrency)
+ else:
+ raise Fail("Couldn't retrieve Hive Server interactive's 'hive.server2.tez.sessions.per.default.queue' config.")
+
+ total_am_capacity_required = self._normalizeUp(llap_daemon_container_size, yarn_min_container_size) \
+ * llap_concurrency + self.calculate_slider_am_size(yarn_min_container_size)
+ cap_available_for_daemons = total_llap_queue_size - total_am_capacity_required
+ if cap_available_for_daemons < yarn_min_container_size :
+ raise Fail("'Capacity available for LLAP daemons'({0}) < 'YARN minimum container size'({1}). Invalid configuration detected. "
+ "Increase LLAP queue size.".format(cap_available_for_daemons, yarn_min_container_size))
+ assert (
+ cap_available_for_daemons > 0), "'Capacity available for daemons' calculated value : {0}. Expected value : > 0".format(
+ cap_available_for_daemons)
+ Logger.info("Calculated cap_available_for_daemons : {0}, using following : llap_slider_cap_percentage : {1}, "
+ "yarn_rm_mem_in_mb : {2}, total_cluster_cap : {3}, total_llap_queue_size : {4}, llap_daemon_container_size"
+ " : {5}, yarn_min_container_size : {6}, llap_concurrency : {7}, total_am_capacity_required : {8}, "
+ .format(cap_available_for_daemons, llap_slider_cap_percentage, yarn_rm_mem_in_mb, total_cluster_cap,
+ total_llap_queue_size, llap_daemon_container_size, yarn_min_container_size, llap_concurrency,
+ total_am_capacity_required))
+ return cap_available_for_daemons
+
+ """
+ Calculates the Slider App Master size based on YARN's Minimum Container Size.
+ """
+ def calculate_slider_am_size(self, yarn_min_container_size):
+ if yarn_min_container_size > 1024:
+ return 1024
+ if yarn_min_container_size > 256 and yarn_min_container_size < 1024:
+ return yarn_min_container_size
+ if yarn_min_container_size < 256:
+ return 256
+
+ """
+ Gets YARN Resource Manager memory in MB (yarn.nodemanager.resource.memory-mb).
+ """
+ def get_yarn_rm_mem_in_mb(self, services):
+ if 'yarn-site' in services['configurations'] and \
+ 'yarn.nodemanager.resource.memory-mb' in services['configurations']['yarn-site']['properties']:
+ yarn_rm_mem_in_mb = float(
+ services['configurations']['yarn-site']['properties']['yarn.nodemanager.resource.memory-mb'])
+ assert (
+ yarn_rm_mem_in_mb > 0.0), "'yarn.nodemanager.resource.memory-mb' current value : {0}. Expected value : > 0".format(
+ yarn_rm_mem_in_mb)
+ else:
+ raise Fail(
+ "Couldn't retrieve YARN's 'yarn.nodemanager.resource.memory-mb' config.")
+ return yarn_rm_mem_in_mb
+
+ """
+ Gets HIVE App Master container size (tez.am.resource.memory.mb)
+ """
+ def get_hive_am_container_size(self, services):
+ llap_daemon_container_size = 0
+ if self.HIVE_INTERACTIVE_SITE in services['configurations'] and \
+ 'tez.am.resource.memory.mb' in services['configurations']['tez-interactive-site']['properties']:
+ llap_daemon_container_size = float(
+ services['configurations']['tez-interactive-site']['properties']['tez.am.resource.memory.mb'])
+ assert (
+ llap_daemon_container_size > 0), "'tez.am.resource.memory.mb' current value : {0}. Expected value : > 0".format(
+ llap_daemon_container_size)
+ else:
+ raise Fail("Couldn't retrieve Hive Server interactive's 'tez.am.resource.memory.mb' config.")
+ return llap_daemon_container_size
+
+
+ """
+ Calculates suggested and maximum value for number of LLAP executors.
+ """
+ def calculate_llap_daemon_executors_count(self, services, llap_container_size):
+ cpu_per_nm_host = 0
+ exec_to_cache_ratio = 1.5
+
+ hive_container_size = self.get_tez_container_size(services)
+
+ if 'yarn.nodemanager.resource.cpu-vcores' in services['configurations']['yarn-site']['properties']:
+ cpu_per_nm_host = float(services['configurations']['yarn-site']['properties'][
+ 'yarn.nodemanager.resource.cpu-vcores'])
+ assert (cpu_per_nm_host > 0), "'yarn.nodemanager.resource.cpu-vcores' current value : {0}. Expected value : > 0"\
+ .format(cpu_per_nm_host)
+ else:
+ raise Fail("Couldn't retrieve YARN's 'yarn.nodemanager.resource.cpu-vcores' config.")
+
+ mem_per_executor = hive_container_size * exec_to_cache_ratio;
+ if mem_per_executor > llap_container_size:
+ mem_per_executor = llap_container_size
+
+ num_executors_per_node_raw = math.floor(llap_container_size / mem_per_executor)
+ num_executors_per_node = min(num_executors_per_node_raw, cpu_per_nm_host)
+ # Allow 4x over-subscription of CPU as a max value
+ num_executors_per_node_max_limit = min(num_executors_per_node_raw, 4 * cpu_per_nm_host)
+ Logger.info("calculated num_executors_per_node: {0}, num_executors_per_node_max_limit : {1}, using following "
+ ": hive_container_size : {2}, cpu_per_nm_host : {3}, mem_per_executor : {4}, num_executors_per_node_raw : {5}"
+ .format(num_executors_per_node, num_executors_per_node_max_limit, hive_container_size,
+ cpu_per_nm_host, mem_per_executor, num_executors_per_node_raw))
+ return num_executors_per_node, num_executors_per_node_max_limit
+
+
+ """
+ Calculates suggested and maximum value for LLAP cache size per node.
+ """
+ def calculate_llap_cache_size_per_executor(self, services, llap_container_size, num_executors_per_node):
+ hive_container_size = self.get_tez_container_size(services)
+ cache_size_per_node = llap_container_size - (num_executors_per_node * hive_container_size)
+ # Reserved memory for minExecutors, which is 1.
+ cache_size_per_node_max_limit = llap_container_size - (1 * hive_container_size)
+ Logger.info("Calculated cache_size_per_node : {0}, cache_size_per_node_max_limit : {1}, using following : "
+ "hive_container_size : {2}, llap_container_size : {3}, num_executors_per_node : {4}"
+ .format(cache_size_per_node, cache_size_per_node_max_limit, hive_container_size, llap_container_size,
+ num_executors_per_node))
+ return cache_size_per_node, cache_size_per_node_max_limit
+
+
+ """
+ Calculates recommended heap size for LLAP app.
+ """
+ def calculate_llap_app_heap_size(self, services, num_executors_per_node):
+ hive_container_size = self.get_tez_container_size(services)
+ total_mem_for_executors = num_executors_per_node * hive_container_size
+ llap_app_heap_size = max(total_mem_for_executors * 0.8, total_mem_for_executors - 1024)
+ Logger.info("Calculated llap_app_heap_size : {0}, using following : hive_container_size : {1}, "
+ "total_mem_for_executors : {2}".format(llap_app_heap_size, hive_container_size, total_mem_for_executors))
+ return max(total_mem_for_executors * 0.8, total_mem_for_executors - 1024)
+
+
+ def _normalizeDown(self, val1, val2):
+ tmp = math.floor(val1 / val2);
+ return tmp * val2;
+
+ def _normalizeUp(self, val1, val2):
+ numContainers = math.ceil(val1 / val2)
+ return numContainers * val2
"""
Checks and (1). Creates 'llap' queue if only 'default' queue exist at leaf level and is consuming 100% capacity OR
@@ -183,7 +663,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
def checkAndManageLlapQueue(self, services, configurations, llap_queue_name):
DEFAULT_LLAP_QUEUE_CAP_PERCENT = 20
putHiveInteractiveEnvProperty = self.putProperty(configurations, "hive-interactive-env", services)
- putHiveInteractiveSiteProperty = self.putProperty(configurations, "hive-interactive-site", services)
+ putHiveInteractiveSiteProperty = self.putProperty(configurations, self.HIVE_INTERACTIVE_SITE, services)
putCapSchedProperty = self.putProperty(configurations, "capacity-scheduler", services)
capacitySchedulerProperties = self.getCapacitySchedulerProperties(services)
@@ -191,16 +671,15 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
if capacitySchedulerProperties:
# Get the llap Cluster percentage used for 'llap' Queue creation
if 'llap_queue_capacity' in services['configurations']['hive-interactive-env']['properties']:
- llapSliderCapPercentage = int(
+ llap_slider_cap_percentage = int(
services['configurations']['hive-interactive-env']['properties']['llap_queue_capacity'])
- if llapSliderCapPercentage not in range(1,101):
- Logger.debug("Adjusting HIVE 'llap_queue_capacity' from {0}% to {1}%".format(llapSliderCapPercentage, DEFAULT_LLAP_QUEUE_CAP_PERCENT))
- llapSliderCapPercentage = DEFAULT_LLAP_QUEUE_CAP_PERCENT # Set the default value to 20.
- putHiveInteractiveEnvProperty('llap_queue_capacity', llapSliderCapPercentage)
+ if llap_slider_cap_percentage not in range(1,101):
+ Logger.info("Adjusting HIVE 'llap_queue_capacity' from {0}% to {1}%".format(llap_slider_cap_percentage, DEFAULT_LLAP_QUEUE_CAP_PERCENT))
+ llap_slider_cap_percentage = DEFAULT_LLAP_QUEUE_CAP_PERCENT # Set the default value to 20.
+ putHiveInteractiveEnvProperty('llap_queue_capacity', llap_slider_cap_percentage)
else:
Logger.error("Problem retrieving LLAP Queue Capacity. Skipping creating {0} queue".format(llap_queue_name))
return
-
leafQueueNames = self.getAllYarnLeafQueues(capacitySchedulerProperties)
capSchedConfigKeys = capacitySchedulerProperties.keys()
@@ -218,21 +697,29 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
if 'yarn.scheduler.capacity.root.'+llap_queue_name+'.capacity' in capSchedConfigKeys:
currLlapQueueCap = capacitySchedulerProperties.get('yarn.scheduler.capacity.root.'+llap_queue_name+'.capacity')
+ if self.HIVE_INTERACTIVE_SITE in services['configurations'] and \
+ 'hive.llap.daemon.queue.name' in services['configurations'][self.HIVE_INTERACTIVE_SITE]['properties']:
+ llap_daemon_selected_queue_name = services['configurations'][self.HIVE_INTERACTIVE_SITE]['properties']['hive.llap.daemon.queue.name']
+ else:
+ Logger.debug("Couldn't retrive 'hive.llap.daemon.queue.name' property. Skipping adjusting queues.")
+ return
updated_cap_sched_configs = ''
+ enabled_hive_int_in_changed_configs = self.are_config_props_in_changed_configs(services, "hive-interactive-env",
+ set(['enable_hive_interactive']), False)
"""
We create OR "modify 'llap' queue 'state and/or capacity' " based on below conditions:
- if only 1 queue exists at root level and is 'default' queue and has 100% cap -> Create 'llap' queue, OR
- if 2 queues exists at root level ('llap' and 'default') :
- - 'llap' queue state is STOPPED -> Modify 'llap' queue state to RUNNING, adjust capacity, OR
- - 'llap' queue state is RUNNING and 'llap_queue_capacity' prop != 'llap' queue current running capacity ->
+ - Queue selected is 'llap' and state is STOPPED -> Modify 'llap' queue state to RUNNING, adjust capacity, OR
+ - Queue selected is 'llap', state is RUNNING and 'llap_queue_capacity' prop != 'llap' queue current running capacity ->
Modify 'llap' queue capacity to 'llap_queue_capacity'
"""
if 'default' in leafQueueNames and \
((len(leafQueueNames) == 1 and int(yarn_default_queue_capacity) == 100) or \
((len(leafQueueNames) == 2 and llap_queue_name in leafQueueNames) and \
- (currLlapQueueState == 'STOPPED' or (currLlapQueueState == 'RUNNING' and currLlapQueueCap != llapSliderCapPercentage)))):
- adjusted_default_queue_cap = str(int(yarn_default_queue_capacity) - llapSliderCapPercentage)
+ ((currLlapQueueState == 'STOPPED' and enabled_hive_int_in_changed_configs) or (currLlapQueueState == 'RUNNING' and currLlapQueueCap != llap_slider_cap_percentage)))):
+ adjusted_default_queue_cap = str(int(yarn_default_queue_capacity) - llap_slider_cap_percentage)
for prop, val in capacitySchedulerProperties.items():
if llap_queue_name not in prop:
if prop == 'yarn.scheduler.capacity.root.queues':
@@ -247,7 +734,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
elif prop.startswith('yarn.') and '.llap.' not in prop:
updated_cap_sched_configs = updated_cap_sched_configs + prop + "=" + val + "\n"
- llapSliderCapPercentage = str(llapSliderCapPercentage)
+ llap_slider_cap_percentage = str(llap_slider_cap_percentage)
hive_user = '*' # Open to all
if 'hive_user' in services['configurations']['hive-env']['properties']:
hive_user = services['configurations']['hive-env']['properties']['hive_user']
@@ -259,9 +746,9 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
+ "yarn.scheduler.capacity.root." + llap_queue_name + ".ordering-policy=fifo\n" \
+ "yarn.scheduler.capacity.root." + llap_queue_name + ".minimum-user-limit-percent=100\n" \
+ "yarn.scheduler.capacity.root." + llap_queue_name + ".maximum-capacity=" \
- + llapSliderCapPercentage + "\n" \
+ + llap_slider_cap_percentage + "\n" \
+ "yarn.scheduler.capacity.root." + llap_queue_name + ".capacity=" \
- + llapSliderCapPercentage + "\n" \
+ + llap_slider_cap_percentage + "\n" \
+ "yarn.scheduler.capacity.root." + llap_queue_name + ".acl_submit_applications=" \
+ hive_user + "\n" \
+ "yarn.scheduler.capacity.root." + llap_queue_name + ".acl_administer_queue=" \
@@ -272,9 +759,9 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
putCapSchedProperty("capacity-scheduler", updated_cap_sched_configs)
if len(leafQueueNames) == 1: # 'llap' queue didn't exist before
Logger.info("Created YARN Queue : '{0}' with capacity : {1}%. Adjusted default queue capacity to : {2}%" \
- .format(llap_queue_name, llapSliderCapPercentage, adjusted_default_queue_cap))
+ .format(llap_queue_name, llap_slider_cap_percentage, adjusted_default_queue_cap))
else: # Queue existed, only adjustments done.
- Logger.info("Adjusted YARN Queue : '{0}'. Current capacity : {1}%. State: RUNNING.".format(llap_queue_name, llapSliderCapPercentage))
+ Logger.info("Adjusted YARN Queue : '{0}'. Current capacity : {1}%. State: RUNNING.".format(llap_queue_name, llap_slider_cap_percentage))
Logger.info("Adjusted 'default' queue capacity to : {0}%".format(adjusted_default_queue_cap))
# Update Hive 'hive.llap.daemon.queue.name' prop to use 'llap' queue.
@@ -296,10 +783,8 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
"""
def checkAndStopLlapQueue(self, services, configurations, llap_queue_name):
putCapSchedProperty = self.putProperty(configurations, "capacity-scheduler", services)
- putHiveInteractiveSiteProperty = self.putProperty(configurations, "hive-interactive-site", services)
-
+ putHiveInteractiveSiteProperty = self.putProperty(configurations, self.HIVE_INTERACTIVE_SITE, services)
capacitySchedulerProperties = self.getCapacitySchedulerProperties(services)
-
updated_default_queue_configs = ''
updated_llap_queue_configs = ''
if capacitySchedulerProperties:
@@ -314,7 +799,6 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
else:
Logger.error("{0} queue 'state' property not present in capacity scheduler. Skipping adjusting queues.".format(llap_queue_name))
return
-
if currLlapQueueState == 'RUNNING':
DEFAULT_MAX_CAPACITY = '100'
for prop, val in capacitySchedulerProperties.items():
@@ -362,7 +846,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
Checks and sets the 'Hive Server Interactive' 'hive.llap.daemon.queue.name' config.
"""
def setLlapDaemonQueueName(self, services, configurations):
- putHiveInteractiveSitePropertyAttribute = self.putPropertyAttribute(configurations, "hive-interactive-site")
+ putHiveInteractiveSitePropertyAttribute = self.putPropertyAttribute(configurations, self.HIVE_INTERACTIVE_SITE)
capacitySchedulerProperties = dict()
# Read 'capacity-scheduler' from configurations if we modified and added recommendation to it, as part of current
@@ -493,4 +977,4 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
hosts_for_component = []
if serviceName in servicesList:
hosts_for_component = [component["hostnames"] for component in componentsList if component["component_name"] == componentName][0]
- return hosts_for_component
+ return hosts_for_component
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/29179e76/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py b/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py
index 44759a9..5712995 100644
--- a/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py
+++ b/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py
@@ -439,7 +439,7 @@ class TestHiveServerInteractive(RMFTestCase):
mock_get_llap_app_status_data.return_value = llap_app_json
status = self.hsi.check_llap_app_status(self.llap_app_name, -1)
- self.assertEqual(status, True)
+ self.assertEqual(status, False)
@patch("time.sleep")
@patch('hive_server_interactive.HiveServerInteractiveDefault._get_llap_app_status_info')
@@ -450,7 +450,7 @@ class TestHiveServerInteractive(RMFTestCase):
mock_get_llap_app_status_data.return_value = llap_app_json
status = self.hsi.check_llap_app_status(self.llap_app_name, 0)
- self.assertEqual(status, True)
+ self.assertEqual(status, False)
@patch("time.sleep")
@patch('hive_server_interactive.HiveServerInteractiveDefault._get_llap_app_status_info')
@@ -461,7 +461,7 @@ class TestHiveServerInteractive(RMFTestCase):
mock_get_llap_app_status_data.return_value = llap_app_json
status = self.hsi.check_llap_app_status(self.llap_app_name, 2)
- self.assertEqual(status, True)
+ self.assertEqual(status, False)