You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sw...@apache.org on 2016/06/24 07:09:43 UTC
[2/2] ambari git commit: AMBARI-17418. Fix for LLAP calculation to
happen on invocation by Blueprints and HIVE SERVER INTERACTIVE is ON (1st
invocation to Stack Advisor). Also, reverts AMBARI-17283.
AMBARI-17418. Fix for LLAP calculation to happen on invocation by Blueprints and HIVE SERVER INTERACTIVE is ON (1st invocation to Stack Advisor). Also, reverts AMBARI-17283.
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b79806ec
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b79806ec
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b79806ec
Branch: refs/heads/trunk
Commit: b79806ec23350ca5ce293e447cd14f7bba13d1a5
Parents: cc9d625
Author: Swapan Shridhar <ss...@hortonworks.com>
Authored: Thu Jun 23 21:48:37 2016 -0700
Committer: Swapan Shridhar <ss...@hortonworks.com>
Committed: Thu Jun 23 23:13:15 2016 -0700
----------------------------------------------------------------------
.../HIVE/configuration/hive-interactive-env.xml | 29 --
.../configuration/hive-interactive-site.xml | 80 ----
.../HDP/2.5/services/HIVE/themes/theme.json | 24 +-
.../stacks/HDP/2.5/services/stack_advisor.py | 447 +++++++++----------
.../stacks/2.5/common/test_stack_advisor.py | 103 +----
5 files changed, 228 insertions(+), 455 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/b79806ec/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml
index c2e1c0a..f2a8d69 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml
@@ -217,35 +217,6 @@
</property>
- <!--
- Note: This config is added for displaying 'num_llap_nodes' value as a 'label' on Hive Server Interactive panel and will always
- have same value as 'num_llap_nodes'. 'num_llap_nodes' will stay editable in Advanced section.
- -->
-
- <property>
- <name>num_llap_nodes_copy</name>
- <value>1</value>
- <description>The number of Hive LLAP daemons to run.</description>
- <display-name>Number of LLAP Daemons</display-name>
- <value-attributes>
- <type>int</type>
- <minimum>1</minimum>
- </value-attributes>
- <depends-on>
- <property>
- <type>hive-interactive-env</type>
- <name>enable_hive_interactive</name>
- </property>
- <property>
- <type>hive-interactive-env</type>
- <name>num_llap_nodes</name>
- </property>
- </depends-on>
- <on-ambari-upgrade add="true"/>
- </property>
-
-
-
<!-- hive-env.sh -->
<property>
<name>content</name>
http://git-wip-us.apache.org/repos/asf/ambari/blob/b79806ec/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
index a0076b3..45d94cd 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
@@ -571,84 +571,4 @@ limitations under the License.
<on-ambari-upgrade add="true"/>
</property>
- <!--
- Note: Properties added here are just for displaying them as 'label' on Hive Server Interactive panel.
- -->
- <property>
- <name>hive.llap.daemon.num.executors_copy</name>
- <!--
- Added for displaying 'hive.llap.daemon.num.executors' value as a 'label' on Hive Server Interactive panel and will always
- have same value as 'hive.llap.daemon.num.executors'. 'hive.llap.daemon.num.executors_copy' will stay editable in Advanced section.
- -->
- <display-name>Maximum CPUs per Daemon</display-name>
- <description>The maximum number of CPUs a single LLAP daemon will use. Usually this should be equal to the number of available CPUs.</description>
- <value>1</value>
- <value-attributes>
- <type>int</type>
- </value-attributes>
- <depends-on>
- <property>
- <type>hive-interactive-site</type>
- <name>hive.llap.daemon.num.executors</name>
- </property>
- <property>
- <type>hive-interactive-env</type>
- <name>enable_hive_interactive</name>
- </property>
- </depends-on>
- <on-ambari-upgrade add="true"/>
- </property>
- <property>
- <name>hive.llap.daemon.yarn.container.mb_copy</name>
- <!--
- Added for displaying 'hive.llap.daemon.yarn.container.mb' value as a 'label' on Hive Server Interactive panel and will always
- have same value as 'hive.llap.daemon.yarn.container.mb'. 'hive.llap.daemon.yarn.container.mb' will stay editable in Advanced section.
- -->
- <display-name>Memory per daemon</display-name>
- <description>Total memory used by individual LLAP daemons. This includes memory for the cache as well as for the query execution.</description>
- <value>341</value>
- <value-attributes>
- <type>int</type>
- <unit>MB</unit>
- <overridable>false</overridable>
- </value-attributes>
- <depends-on>
- <property>
- <type>hive-interactive-env</type>
- <name>enable_hive_interactive</name>
- </property>
- <property>
- <type>hive-interactive-site</type>
- <name>hive.llap.daemon.yarn.container.mb</name>
- </property>
- </depends-on>
- <on-ambari-upgrade add="true"/>
- </property>
- <property>
- <name>hive.llap.io.memory.size_copy</name>
- <!--
- Added for displaying 'hive.llap.io.memory.size' value as a 'label' on Hive Server Interactive panel and will always
- have same value as 'hive.llap.io.memory.size'. 'hive.llap.io.memory.size' will stay editable in Advanced section.
- -->
- <display-name>In-Memory Cache per Daemon</display-name>
- <description>The amount of memory reserved for Hive's optimized in-memory cache.</description>
- <value>0</value>
- <value-attributes>
- <type>int</type>
- <unit>MB</unit>
- <overridable>false</overridable>
- </value-attributes>
- <depends-on>
- <property>
- <type>hive-interactive-env</type>
- <name>enable_hive_interactive</name>
- </property>
- <property>
- <type>hive-interactive-site</type>
- <name>hive.llap.io.memory.size</name>
- </property>
- </depends-on>
- <on-ambari-upgrade add="true"/>
- </property>
-
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/b79806ec/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json
index 81277a8..c6cb873 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json
@@ -89,7 +89,7 @@
]
},
{
- "config": "hive-interactive-env/num_llap_nodes_copy",
+ "config": "hive-interactive-env/num_llap_nodes",
"subsection-name": "interactive-query-row1-col1",
"depends-on": [
{
@@ -111,7 +111,7 @@
]
},
{
- "config": "hive-interactive-site/hive.llap.daemon.yarn.container.mb_copy",
+ "config": "hive-interactive-site/hive.llap.daemon.yarn.container.mb",
"subsection-name": "interactive-query-row1-col1",
"depends-on": [
{
@@ -133,7 +133,7 @@
]
},
{
- "config": "hive-interactive-site/hive.llap.io.memory.size_copy",
+ "config": "hive-interactive-site/hive.llap.io.memory.size",
"subsection-name": "interactive-query-row1-col1",
"depends-on": [
{
@@ -155,7 +155,7 @@
]
},
{
- "config": "hive-interactive-site/hive.llap.daemon.num.executors_copy",
+ "config": "hive-interactive-site/hive.llap.daemon.num.executors",
"subsection-name": "interactive-query-row1-col1",
"depends-on": [
{
@@ -210,9 +210,9 @@
}
},
{
- "config": "hive-interactive-env/num_llap_nodes_copy",
+ "config": "hive-interactive-env/num_llap_nodes",
"widget": {
- "type": "label",
+ "type": "text-field",
"units": [
{
"unit-name": "int"
@@ -232,9 +232,9 @@
}
},
{
- "config": "hive-interactive-site/hive.llap.daemon.yarn.container.mb_copy",
+ "config": "hive-interactive-site/hive.llap.daemon.yarn.container.mb",
"widget": {
- "type": "label",
+ "type": "text-field",
"units": [
{
"unit-name": "MB"
@@ -243,9 +243,9 @@
}
},
{
- "config": "hive-interactive-site/hive.llap.io.memory.size_copy",
+ "config": "hive-interactive-site/hive.llap.io.memory.size",
"widget": {
- "type": "label",
+ "type": "text-field",
"units": [
{
"unit-name": "MB"
@@ -254,9 +254,9 @@
}
},
{
- "config": "hive-interactive-site/hive.llap.daemon.num.executors_copy",
+ "config": "hive-interactive-site/hive.llap.daemon.num.executors",
"widget": {
- "type": "label",
+ "type": "text-field",
"units": [
{
"unit-name": "int"
http://git-wip-us.apache.org/repos/asf/ambari/blob/b79806ec/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
index 6b1d631..af2d2c1 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
@@ -402,32 +402,12 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
putHiveInteractiveEnvProperty('enable_hive_interactive', 'false')
putHiveInteractiveEnvPropertyAttribute("llap_queue_capacity", "visible", "false")
- if self.HIVE_INTERACTIVE_SITE in services['configurations']:
- if 'hive.llap.zk.sm.connectionString' in services['configurations'][self.HIVE_INTERACTIVE_SITE]['properties']:
- # Fill the property 'hive.llap.zk.sm.connectionString' required by Hive Server Interactive (HiveServer2)
- zookeeper_host_port = self.getZKHostPortString(services)
- if zookeeper_host_port:
- putHiveInteractiveSiteProperty("hive.llap.zk.sm.connectionString", zookeeper_host_port)
-
- # Copy the value of 'hive.llap.daemon.yarn.container.mb', 'hive.llap.daemon.num.executors', 'hive.llap.io.memory.size' and 'num_llap_nodes'
- # to their '*_copy' configs as they are used to display the values as 'label' on Hive Server interactive panel.
- if 'hive.llap.daemon.yarn.container.mb' in services['configurations'][self.HIVE_INTERACTIVE_SITE]['properties']:
- hive_llap_yarn_cont_mb = services['configurations'][self.HIVE_INTERACTIVE_SITE]['properties']['hive.llap.daemon.yarn.container.mb']
- putHiveInteractiveSiteProperty("hive.llap.daemon.yarn.container.mb_copy", hive_llap_yarn_cont_mb)
- Logger.info("Adjusted config 'hive.llap.daemon.yarn.container.mb_copy' value same as config 'hive.llap.daemon.yarn.container.mb' value({0})".format(hive_llap_yarn_cont_mb))
- if 'hive.llap.daemon.num.executors' in services['configurations'][self.HIVE_INTERACTIVE_SITE]['properties']:
- hive_llap_daemon_num_executors = services['configurations'][self.HIVE_INTERACTIVE_SITE]['properties']['hive.llap.daemon.num.executors']
- putHiveInteractiveSiteProperty("hive.llap.daemon.num.executors_copy", hive_llap_daemon_num_executors)
- Logger.info("Adjusted config 'hive.llap.daemon.num.executors_copy' value same as config 'hive.llap.daemon.num.executors' value({0})".format(hive_llap_daemon_num_executors))
- if 'hive.llap.io.memory.size' in services['configurations'][self.HIVE_INTERACTIVE_SITE]['properties']:
- hive_llap_io_mem_size = services['configurations'][self.HIVE_INTERACTIVE_SITE]['properties']['hive.llap.io.memory.size']
- putHiveInteractiveSiteProperty("hive.llap.io.memory.size_copy", hive_llap_io_mem_size)
- Logger.info("Adjusted config 'hive.llap.io.memory.size_copy' value same as config 'hive.llap.io.memory.size' value({0})".format(hive_llap_io_mem_size))
- if 'hive-interactive-env' in services['configurations'] and \
- 'num_llap_nodes' in services['configurations']['hive-interactive-env']['properties']:
- num_llap_node = services['configurations']['hive-interactive-env']['properties']['num_llap_nodes']
- putHiveInteractiveEnvProperty("num_llap_nodes_copy", num_llap_node)
- Logger.info("Adjusted config 'num_llap_nodes_copy' value same as config 'num_llap_nodes' value({0})".format(num_llap_node))
+ if self.HIVE_INTERACTIVE_SITE in services['configurations'] and \
+ 'hive.llap.zk.sm.connectionString' in services['configurations'][self.HIVE_INTERACTIVE_SITE]['properties']:
+ # Fill the property 'hive.llap.zk.sm.connectionString' required by Hive Server Interactive (HiveServer2)
+ zookeeper_host_port = self.getZKHostPortString(services)
+ if zookeeper_host_port:
+ putHiveInteractiveSiteProperty("hive.llap.zk.sm.connectionString", zookeeper_host_port)
pass
def recommendYARNConfigurations(self, configurations, clusterData, services, hosts):
@@ -447,10 +427,9 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
self.checkAndStopLlapQueue(services, configurations, LLAP_QUEUE_NAME)
"""
- Entry point for updating Hive's 'LLAP app' configs namely : (1). num_llap_nodes, num_llap_nodes_copy
- (2). hive.llap.daemon.yarn.container.mb, hive.llap.daemon.yarn.container.mb_copy (3). hive.llap.daemon.num.executors,
- hive.llap.daemon.num.executors_copy (4). hive.llap.io.memory.size, hive.llap.io.memory.size_copy (5). llap_heap_size
- (6). slider_am_container_mb, and (7). hive.server2.tez.sessions.per.default.queue
+ Entry point for updating Hive's 'LLAP app' configs namely : (1). num_llap_nodes (2). hive.llap.daemon.yarn.container.mb
+ (3). hive.llap.daemon.num.executors (4). hive.llap.io.memory.size (5). llap_heap_size (6). slider_am_container_mb,
+ and (7). hive.server2.tez.sessions.per.default.queue
The trigger point for updating LLAP configs (mentioned above) is change in values of any of the following:
(1). 'enable_hive_interactive' set to 'true' (2). 'llap_queue_capacity' (3). 'hive.server2.tez.sessions.per.default.queue'
@@ -458,10 +437,6 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
If change in value for 'llap_queue_capacity' or 'hive.server2.tez.sessions.per.default.queue' is detected, that config
value is not calulated, but read and use in calculation for dependent configs.
-
- Note : Configs with name as '*_copy' will have same value as their conterpart configs (eg: 'num_llap_nodes'
- for 'num_llap_nodes_copy'). '*_copy' configs are added so that we can show them as label on UI in Hive Server Interactive
- Panel and their counterparts shown in 'Advanced' section as editable ones.
"""
def updateLlapConfigs(self, configurations, services, hosts, llap_queue_name):
putHiveInteractiveSiteProperty = self.putProperty(configurations, self.HIVE_INTERACTIVE_SITE, services)
@@ -484,15 +459,6 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
# initial memory setting to make sure hive.llap.daemon.yarn.container.mb >= yarn.scheduler.minimum-allocation-mb
Logger.debug("Adjusted 'hive.llap.daemon.yarn.container.mb' to yarn min container size as initial size "
"(" + str(self.get_yarn_min_container_size(services, configurations)) + " MB).")
- # This is a stopgap thing, to be sure that on 1st invocation, value for 'hive.llap.daemon.yarn.container.mb_copy'
- # is same as value set by 'hive.llap.daemon.yarn.container.mb'.
- if 'hive-interactive-site' not in configurations or \
- ('hive-interactive-site' in configurations and \
- 'hive.llap.daemon.yarn.container.mb_copy' not in configurations['hive-interactive-site']['properties']):
- putHiveInteractiveSiteProperty('hive.llap.daemon.yarn.container.mb_copy', read_llap_daemon_yarn_cont_mb)
- Logger.debug("Adjusted 'hive.llap.daemon.yarn.container.mb_copy' to yarn min container size as initial size "
- "(" + str(self.get_yarn_min_container_size(services, configurations)) + " MB).")
-
try:
if self.HIVE_INTERACTIVE_SITE in services['configurations'] and \
@@ -528,10 +494,16 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
Logger.error("Couldn't retrieve 'capacity-scheduler' properties while doing YARN queue adjustment for Hive Server Interactive."
" Not calculating LLAP configs.")
return
+
+ changed_configs_in_hive_int_env = None
+ llap_concurrency_in_changed_configs = None
+ llap_daemon_queue_in_changed_configs = None
+ # Calculations are triggered only if there is change in any one of the following props :
+ # 'llap_queue_capacity', 'enable_hive_interactive', 'hive.server2.tez.sessions.per.default.queue'
+ # or 'hive.llap.daemon.queue.name' has change in value selection.
+ # OR
+ # services['changed-configurations'] is empty implying that this is the Blueprint call. (1st invocation)
if 'changed-configurations' in services.keys():
- # Calculations are triggered only if there is change in any one of the following props :
- # 'llap_queue_capacity', 'enable_hive_interactive', 'hive.server2.tez.sessions.per.default.queue'
- # or 'hive.llap.daemon.queue.name' has change in value selection.
config_names_to_be_checked = set(['llap_queue_capacity', 'enable_hive_interactive'])
changed_configs_in_hive_int_env = self.are_config_props_in_changed_configs(services, "hive-interactive-env",
config_names_to_be_checked, False)
@@ -542,191 +514,200 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
llap_daemon_queue_in_changed_configs = self.are_config_props_in_changed_configs(services, "hive-interactive-site",
set(['hive.llap.daemon.queue.name']), False)
- if not changed_configs_in_hive_int_env and \
- not llap_concurrency_in_changed_configs and \
- not llap_daemon_queue_in_changed_configs:
-
- Logger.info("LLAP parameters not modified. Not adjusting LLAP configs.")
- Logger.debug("Current 'changed-configuration' received is : {0}".format(services["changed-configurations"]))
- return
+ if not changed_configs_in_hive_int_env and \
+ not llap_concurrency_in_changed_configs and \
+ not llap_daemon_queue_in_changed_configs and \
+ services["changed-configurations"]:
+ Logger.info("LLAP parameters not modified. Not adjusting LLAP configs.")
+ Logger.info("Current 'changed-configuration' received is : {0}".format(services["changed-configurations"]))
+ return
- node_manager_host_list = self.get_node_manager_hosts(services, hosts)
- node_manager_cnt = len(node_manager_host_list)
- # Check which queue is selected in 'hive.llap.daemon.queue.name', to determine current queue capacity
- current_selected_queue_for_llap_cap = None
- if llap_queue_selected_in_current_call == llap_queue_name or llap_daemon_selected_queue_name == llap_queue_name:
- current_selected_queue_for_llap_cap = self.get_llap_cap_percent_slider(services, configurations)
- else: # any queue other than 'llap'
- current_selected_queue_for_llap_cap = capacity_scheduler_properties.get('yarn.scheduler.capacity.root.'+llap_daemon_selected_queue_name+'.capacity')
-
- assert (current_selected_queue_for_llap_cap >= 1), "Current selected current value : {0}. Expected value : >= 1" \
- .format(current_selected_queue_for_llap_cap)
-
- yarn_nm_mem_in_mb = self.get_yarn_nm_mem_in_mb(services, configurations)
- total_cluster_capacity = node_manager_cnt * yarn_nm_mem_in_mb
- Logger.info("\n\nCalculated total_cluster_capacity : {0}, using following : node_manager_cnt : {1}, "
- "yarn_nm_mem_in_mb : {2}".format(total_cluster_capacity, node_manager_cnt, yarn_nm_mem_in_mb))
-
- yarn_min_container_size = self.get_yarn_min_container_size(services, configurations)
- tez_am_container_size = self.calculate_tez_am_container_size(long(total_cluster_capacity))
- normalized_tez_am_container_size = self._normalizeUp(tez_am_container_size, yarn_min_container_size)
- Logger.info("Calculated normalized_tez_am_container_size : {0}, using following : tez_am_container_size : {1}, "
- "total_cluster_capacity : {2}".format(normalized_tez_am_container_size, tez_am_container_size, total_cluster_capacity))
- total_llap_queue_size = long(self._normalizeDown((float(current_selected_queue_for_llap_cap) / 100 * total_cluster_capacity), yarn_min_container_size))
- # Get calculated value for Slider AM container Size
- slider_am_container_size = self._normalizeUp(self.calculate_slider_am_size(yarn_min_container_size), yarn_min_container_size)
-
- # Read 'hive.server2.tez.sessions.per.default.queue' prop if it's in changed-configs, else calculate it.
- if not llap_concurrency_in_changed_configs:
- # Calculate llap concurrency (i.e. Number of Tez AM's)
- llap_concurrency = float(total_llap_queue_size * 0.25 / normalized_tez_am_container_size)
- llap_concurrency = max(long(llap_concurrency), 1)
- Logger.info("Calculated llap_concurrency : {0}, using following : total_llap_queue_size : {1}, "
- "normalized_tez_am_container_size : {2}".format(llap_concurrency, total_llap_queue_size, normalized_tez_am_container_size))
- # Limit 'llap_concurrency' to reach a max. of 32.
- if llap_concurrency > LLAP_MAX_CONCURRENCY:
- llap_concurrency = LLAP_MAX_CONCURRENCY
- else:
- # Read current value
- if 'hive.server2.tez.sessions.per.default.queue' in services['configurations'][self.HIVE_INTERACTIVE_SITE]['properties']:
- llap_concurrency = long(services['configurations'][self.HIVE_INTERACTIVE_SITE]['properties'][
- 'hive.server2.tez.sessions.per.default.queue'])
- assert (llap_concurrency >= 1), "'hive.server2.tez.sessions.per.default.queue' current value : {0}. Expected value : >= 1" \
- .format(llap_concurrency)
- else:
- raise Fail("Couldn't retrieve Hive Server interactive's 'hive.server2.tez.sessions.per.default.queue' config.")
-
-
- # Calculate 'total memory available for llap daemons' across cluster
- total_am_capacity_required = normalized_tez_am_container_size * llap_concurrency + slider_am_container_size
- cap_available_for_daemons = total_llap_queue_size - total_am_capacity_required
- Logger.info("Calculated cap_available_for_daemons : {0}, using following : current_selected_queue_for_llap_cap : {1}, "
- "yarn_nm_mem_in_mb : {2}, total_cluster_capacity : {3}, total_llap_queue_size : {4}, normalized_tez_am_container_size"
- " : {5}, yarn_min_container_size : {6}, llap_concurrency : {7}, total_am_capacity_required : {8}"
- .format(cap_available_for_daemons, current_selected_queue_for_llap_cap, yarn_nm_mem_in_mb, total_cluster_capacity,
- total_llap_queue_size, normalized_tez_am_container_size, yarn_min_container_size, llap_concurrency,
- total_am_capacity_required))
- if cap_available_for_daemons < yarn_min_container_size :
- raise Fail("'Capacity available for LLAP daemons'({0}) < 'YARN minimum container size'({1}). Invalid configuration detected. "
- "Increase LLAP queue size.".format(cap_available_for_daemons, yarn_min_container_size))
-
-
-
- # Calculate value for 'num_llap_nodes', an across cluster config.
- # Also, get calculated value for 'hive.llap.daemon.yarn.container.mb' based on 'num_llap_nodes' value, a per node config.
- num_llap_nodes_raw = cap_available_for_daemons / yarn_nm_mem_in_mb
- if num_llap_nodes_raw < 1.00:
- # Set the llap nodes to min. value of 1 and 'llap_container_size' to min. YARN allocation.
- num_llap_nodes = 1
- llap_container_size = self._normalizeUp(cap_available_for_daemons, yarn_min_container_size)
- Logger.info("Calculated llap_container_size : {0}, using following : cap_available_for_daemons : {1}, "
- "yarn_min_container_size : {2}".format(llap_container_size, cap_available_for_daemons, yarn_min_container_size))
- else:
- num_llap_nodes = math.floor(num_llap_nodes_raw)
- llap_container_size = self._normalizeDown(yarn_nm_mem_in_mb, yarn_min_container_size)
- Logger.info("Calculated llap_container_size : {0}, using following : yarn_nm_mem_in_mb : {1}, "
- "yarn_min_container_size : {2}".format(llap_container_size, yarn_nm_mem_in_mb, yarn_min_container_size))
- Logger.info("Calculated num_llap_nodes : {0} using following : yarn_nm_mem_in_mb : {1}, cap_available_for_daemons : {2} " \
- .format(num_llap_nodes, yarn_nm_mem_in_mb, cap_available_for_daemons))
-
-
- # Calculate value for 'hive.llap.daemon.num.executors', a per node config.
- hive_tez_container_size = self.get_hive_tez_container_size(services, configurations)
- if 'yarn.nodemanager.resource.cpu-vcores' in services['configurations']['yarn-site']['properties']:
- cpu_per_nm_host = float(services['configurations']['yarn-site']['properties'][
- 'yarn.nodemanager.resource.cpu-vcores'])
- assert (cpu_per_nm_host > 0), "'yarn.nodemanager.resource.cpu-vcores' current value : {0}. Expected value : > 0" \
- .format(cpu_per_nm_host)
+ node_manager_host_list = self.get_node_manager_hosts(services, hosts)
+ node_manager_cnt = len(node_manager_host_list)
+ # Check which queue is selected in 'hive.llap.daemon.queue.name', to determine current queue capacity
+ current_selected_queue_for_llap_cap = None
+ if llap_queue_selected_in_current_call == llap_queue_name or llap_daemon_selected_queue_name == llap_queue_name:
+ current_selected_queue_for_llap_cap = self.get_llap_cap_percent_slider(services, configurations)
+ else: # any queue other than 'llap'
+ current_selected_queue_for_llap_cap = capacity_scheduler_properties.get(
+ 'yarn.scheduler.capacity.root.' + llap_daemon_selected_queue_name + '.capacity')
+
+ assert (current_selected_queue_for_llap_cap >= 1), "Current selected current value : {0}. Expected value : >= 1" \
+ .format(current_selected_queue_for_llap_cap)
+
+ yarn_nm_mem_in_mb = self.get_yarn_nm_mem_in_mb(services, configurations)
+ total_cluster_capacity = node_manager_cnt * yarn_nm_mem_in_mb
+ Logger.info("\n\nCalculated total_cluster_capacity : {0}, using following : node_manager_cnt : {1}, "
+ "yarn_nm_mem_in_mb : {2}".format(total_cluster_capacity, node_manager_cnt, yarn_nm_mem_in_mb))
+
+ yarn_min_container_size = self.get_yarn_min_container_size(services, configurations)
+ tez_am_container_size = self.calculate_tez_am_container_size(long(total_cluster_capacity))
+ normalized_tez_am_container_size = self._normalizeUp(tez_am_container_size, yarn_min_container_size)
+ Logger.info("Calculated normalized_tez_am_container_size : {0}, using following : tez_am_container_size : {1}, "
+ "total_cluster_capacity : {2}".format(normalized_tez_am_container_size, tez_am_container_size,
+ total_cluster_capacity))
+ total_llap_queue_size = long(
+ self._normalizeDown((float(current_selected_queue_for_llap_cap) / 100 * total_cluster_capacity),
+ yarn_min_container_size))
+ # Get calculated value for Slider AM container Size
+ slider_am_container_size = self._normalizeUp(self.calculate_slider_am_size(yarn_min_container_size),
+ yarn_min_container_size)
+
+ # Read 'hive.server2.tez.sessions.per.default.queue' prop if it's in changed-configs, else calculate it.
+ if not llap_concurrency_in_changed_configs:
+ # Calculate llap concurrency (i.e. Number of Tez AM's)
+ llap_concurrency = float(total_llap_queue_size * 0.25 / normalized_tez_am_container_size)
+ llap_concurrency = max(long(llap_concurrency), 1)
+ Logger.info("Calculated llap_concurrency : {0}, using following : total_llap_queue_size : {1}, "
+ "normalized_tez_am_container_size : {2}".format(llap_concurrency, total_llap_queue_size,
+ normalized_tez_am_container_size))
+ # Limit 'llap_concurrency' to reach a max. of 32.
+ if llap_concurrency > LLAP_MAX_CONCURRENCY:
+ llap_concurrency = LLAP_MAX_CONCURRENCY
+ else:
+ # Read current value
+ if 'hive.server2.tez.sessions.per.default.queue' in services['configurations'][self.HIVE_INTERACTIVE_SITE][
+ 'properties']:
+ llap_concurrency = long(services['configurations'][self.HIVE_INTERACTIVE_SITE]['properties'][
+ 'hive.server2.tez.sessions.per.default.queue'])
+ assert (
+ llap_concurrency >= 1), "'hive.server2.tez.sessions.per.default.queue' current value : {0}. Expected value : >= 1" \
+ .format(llap_concurrency)
else:
- raise Fail("Couldn't retrieve YARN's 'yarn.nodemanager.resource.cpu-vcores' config.")
-
- num_executors_per_node_raw = math.floor(llap_container_size / hive_tez_container_size)
- num_executors_per_node = min(num_executors_per_node_raw, cpu_per_nm_host)
- Logger.info("calculated num_executors_per_node: {0}, using following : hive_tez_container_size : {1}, "
- "cpu_per_nm_host : {2}, num_executors_per_node_raw : {3}, llap_container_size : {4}"
- .format(num_executors_per_node, hive_tez_container_size, cpu_per_nm_host, num_executors_per_node_raw,
- llap_container_size))
- assert (num_executors_per_node >= 0), "'Number of executors per node' : {0}. Expected value : > 0".format(
- num_executors_per_node)
-
- total_mem_for_executors = num_executors_per_node * hive_tez_container_size
-
- # Calculate value for 'cache' (hive.llap.io.memory.size), a per node config.
- cache_size_per_node = llap_container_size - total_mem_for_executors
- Logger.info("Calculated cache_size_per_node : {0} using following : hive_container_size : {1}, llap_container_size"
- " : {2}, num_executors_per_node : {3}"
- .format(cache_size_per_node, hive_tez_container_size, llap_container_size, num_executors_per_node))
- if cache_size_per_node < 0: # Run with '0' cache.
- Logger.info("Calculated 'cache_size_per_node' : {0}. Setting 'cache_size_per_node' to 0.".format(cache_size_per_node))
- cache_size_per_node = 0
-
-
- # Calculate value for prop 'llap_heap_size'
- llap_xmx = max(total_mem_for_executors * 0.8, total_mem_for_executors - 1024)
- Logger.info("Calculated llap_app_heap_size : {0}, using following : hive_container_size : {1}, "
- "total_mem_for_executors : {2}".format(llap_xmx, hive_tez_container_size, total_mem_for_executors))
-
-
- # Updating calculated configs.
- normalized_tez_am_container_size = long(normalized_tez_am_container_size)
- putTezInteractiveSiteProperty('tez.am.resource.memory.mb', normalized_tez_am_container_size)
- Logger.info("'Tez for Hive2' config 'tez.am.resource.memory.mb' updated. Current: {0}".format(normalized_tez_am_container_size))
-
- if not llap_concurrency_in_changed_configs:
- min_llap_concurrency = 1
- putHiveInteractiveSiteProperty('hive.server2.tez.sessions.per.default.queue', llap_concurrency)
- putHiveInteractiveSitePropertyAttribute('hive.server2.tez.sessions.per.default.queue', "minimum", min_llap_concurrency)
- putHiveInteractiveSitePropertyAttribute('hive.server2.tez.sessions.per.default.queue', "maximum", LLAP_MAX_CONCURRENCY)
- Logger.info("Hive2 config 'hive.server2.tez.sessions.per.default.queue' updated. Min : {0}, Current: {1}, Max: {2}" \
- .format(min_llap_concurrency, llap_concurrency, LLAP_MAX_CONCURRENCY))
-
- num_llap_nodes = long(num_llap_nodes)
-
- putHiveInteractiveEnvProperty('num_llap_nodes', num_llap_nodes)
- Logger.info("LLAP config 'num_llap_nodes' updated. Current: {0}".format(num_llap_nodes))
- # Copy the value of 'num_llap_nodes' to 'num_llap_nodes_copy' as 'num_llap_nodes_copy' config is used to display
- # it as Label on Hive Server interactive panel.
- putHiveInteractiveEnvProperty('num_llap_nodes_copy', num_llap_nodes)
-
- llap_container_size = long(llap_container_size)
- putHiveInteractiveSiteProperty('hive.llap.daemon.yarn.container.mb', llap_container_size)
- Logger.info("LLAP config 'hive.llap.daemon.yarn.container.mb' updated. Current: {0}".format(llap_container_size))
- # Copy the value of 'hive.llap.daemon.yarn.container.mb' to 'hive.llap.daemon.yarn.container.mb_copy' as
- # 'hive.llap.daemon.yarn.container.mb_copy' config is used to display it as Label on Hive Server interactive panel.
- putHiveInteractiveSiteProperty('hive.llap.daemon.yarn.container.mb_copy', llap_container_size)
-
- num_executors_per_node = long(num_executors_per_node)
- putHiveInteractiveSiteProperty('hive.llap.daemon.num.executors', num_executors_per_node)
- # Copy the value of 'hive.llap.daemon.num.executors' to 'hive.llap.daemon.num.executors_copy' as
- # 'hive.llap.daemon.num.executors_copy' config is used to display it as Label on Hive Server interactive panel.
- putHiveInteractiveSiteProperty('hive.llap.daemon.num.executors_copy', num_executors_per_node)
- Logger.info("LLAP config 'hive.llap.daemon.num.executors' updated. Current: {0}".format(num_executors_per_node))
- # 'hive.llap.io.threadpool.size' config value is to be set same as value calculated for
- # 'hive.llap.daemon.num.executors' at all times.
- putHiveInteractiveSiteProperty('hive.llap.io.threadpool.size', num_executors_per_node)
- Logger.info("LLAP config 'hive.llap.io.threadpool.size' updated. Current: {0}".format(num_executors_per_node))
-
- cache_size_per_node = long(cache_size_per_node)
- putHiveInteractiveSiteProperty('hive.llap.io.memory.size', cache_size_per_node)
- Logger.info("LLAP config 'hive.llap.io.memory.size' updated. Current: {0}".format(cache_size_per_node))
- # Copy the value of 'hive.llap.io.memory.size' to 'hive.llap.io.memory.size_copy' as
- # 'hive.llap.io.memory.size_copy' config is used to display it as Label on Hive Server interactive panel.
- putHiveInteractiveSiteProperty('hive.llap.io.memory.size_copy', cache_size_per_node)
- llap_io_enabled = 'false'
- if cache_size_per_node >= 64:
- llap_io_enabled = 'true'
-
- putHiveInteractiveSiteProperty('hive.llap.io.enabled', llap_io_enabled)
- Logger.info("Hive2 config 'hive.llap.io.enabled' updated to '{0}' as part of "
- "'hive.llap.io.memory.size' calculation.".format(llap_io_enabled))
-
- llap_xmx = long(llap_xmx)
- putHiveInteractiveEnvProperty('llap_heap_size', llap_xmx)
- Logger.info("LLAP config 'llap_heap_size' updated. Current: {0}".format(llap_xmx))
-
- slider_am_container_size = long(slider_am_container_size)
- putHiveInteractiveEnvProperty('slider_am_container_mb', slider_am_container_size)
- Logger.info("LLAP config 'slider_am_container_mb' updated. Current: {0}".format(slider_am_container_size))
+ raise Fail(
+ "Couldn't retrieve Hive Server interactive's 'hive.server2.tez.sessions.per.default.queue' config.")
+
+
+ # Calculate 'total memory available for llap daemons' across cluster
+ total_am_capacity_required = normalized_tez_am_container_size * llap_concurrency + slider_am_container_size
+ cap_available_for_daemons = total_llap_queue_size - total_am_capacity_required
+ Logger.info(
+ "Calculated cap_available_for_daemons : {0}, using following : current_selected_queue_for_llap_cap : {1}, "
+ "yarn_nm_mem_in_mb : {2}, total_cluster_capacity : {3}, total_llap_queue_size : {4}, normalized_tez_am_container_size"
+ " : {5}, yarn_min_container_size : {6}, llap_concurrency : {7}, total_am_capacity_required : {8}"
+ .format(cap_available_for_daemons, current_selected_queue_for_llap_cap, yarn_nm_mem_in_mb,
+ total_cluster_capacity,
+ total_llap_queue_size, normalized_tez_am_container_size, yarn_min_container_size, llap_concurrency,
+ total_am_capacity_required))
+ if cap_available_for_daemons < yarn_min_container_size:
+ raise Fail(
+ "'Capacity available for LLAP daemons'({0}) < 'YARN minimum container size'({1}). Invalid configuration detected. "
+ "Increase LLAP queue size.".format(cap_available_for_daemons, yarn_min_container_size))
+
+
+
+ # Calculate value for 'num_llap_nodes', an across cluster config.
+ # Also, get calculated value for 'hive.llap.daemon.yarn.container.mb' based on 'num_llap_nodes' value, a per node config.
+ num_llap_nodes_raw = cap_available_for_daemons / yarn_nm_mem_in_mb
+ if num_llap_nodes_raw < 1.00:
+ # Set the llap nodes to min. value of 1 and 'llap_container_size' to min. YARN allocation.
+ num_llap_nodes = 1
+ llap_container_size = self._normalizeUp(cap_available_for_daemons, yarn_min_container_size)
+ Logger.info("Calculated llap_container_size : {0}, using following : cap_available_for_daemons : {1}, "
+ "yarn_min_container_size : {2}".format(llap_container_size, cap_available_for_daemons,
+ yarn_min_container_size))
+ else:
+ num_llap_nodes = math.floor(num_llap_nodes_raw)
+ llap_container_size = self._normalizeDown(yarn_nm_mem_in_mb, yarn_min_container_size)
+ Logger.info("Calculated llap_container_size : {0}, using following : yarn_nm_mem_in_mb : {1}, "
+ "yarn_min_container_size : {2}".format(llap_container_size, yarn_nm_mem_in_mb,
+ yarn_min_container_size))
+ Logger.info(
+ "Calculated num_llap_nodes : {0} using following : yarn_nm_mem_in_mb : {1}, cap_available_for_daemons : {2} " \
+ .format(num_llap_nodes, yarn_nm_mem_in_mb, cap_available_for_daemons))
+
+
+ # Calculate value for 'hive.llap.daemon.num.executors', a per node config.
+ hive_tez_container_size = self.get_hive_tez_container_size(services, configurations)
+ if 'yarn.nodemanager.resource.cpu-vcores' in services['configurations']['yarn-site']['properties']:
+ cpu_per_nm_host = float(services['configurations']['yarn-site']['properties'][
+ 'yarn.nodemanager.resource.cpu-vcores'])
+ assert (cpu_per_nm_host > 0), "'yarn.nodemanager.resource.cpu-vcores' current value : {0}. Expected value : > 0" \
+ .format(cpu_per_nm_host)
+ else:
+ raise Fail("Couldn't retrieve YARN's 'yarn.nodemanager.resource.cpu-vcores' config.")
+
+ num_executors_per_node_raw = math.floor(llap_container_size / hive_tez_container_size)
+ num_executors_per_node = min(num_executors_per_node_raw, cpu_per_nm_host)
+ Logger.info("calculated num_executors_per_node: {0}, using following : hive_tez_container_size : {1}, "
+ "cpu_per_nm_host : {2}, num_executors_per_node_raw : {3}, llap_container_size : {4}"
+ .format(num_executors_per_node, hive_tez_container_size, cpu_per_nm_host, num_executors_per_node_raw,
+ llap_container_size))
+ assert (num_executors_per_node >= 0), "'Number of executors per node' : {0}. Expected value : > 0".format(
+ num_executors_per_node)
+
+ total_mem_for_executors = num_executors_per_node * hive_tez_container_size
+
+ # Calculate value for 'cache' (hive.llap.io.memory.size), a per node config.
+ cache_size_per_node = llap_container_size - total_mem_for_executors
+ Logger.info(
+ "Calculated cache_size_per_node : {0} using following : hive_container_size : {1}, llap_container_size"
+ " : {2}, num_executors_per_node : {3}"
+ .format(cache_size_per_node, hive_tez_container_size, llap_container_size, num_executors_per_node))
+ if cache_size_per_node < 0: # Run with '0' cache.
+ Logger.info(
+ "Calculated 'cache_size_per_node' : {0}. Setting 'cache_size_per_node' to 0.".format(cache_size_per_node))
+ cache_size_per_node = 0
+
+
+ # Calculate value for prop 'llap_heap_size'
+ llap_xmx = max(total_mem_for_executors * 0.8, total_mem_for_executors - 1024)
+ Logger.info("Calculated llap_app_heap_size : {0}, using following : hive_container_size : {1}, "
+ "total_mem_for_executors : {2}".format(llap_xmx, hive_tez_container_size, total_mem_for_executors))
+
+
+ # Updating calculated configs.
+ normalized_tez_am_container_size = long(normalized_tez_am_container_size)
+ putTezInteractiveSiteProperty('tez.am.resource.memory.mb', normalized_tez_am_container_size)
+ Logger.info("'Tez for Hive2' config 'tez.am.resource.memory.mb' updated. Current: {0}".format(
+ normalized_tez_am_container_size))
+
+ if not llap_concurrency_in_changed_configs:
+ min_llap_concurrency = 1
+ putHiveInteractiveSiteProperty('hive.server2.tez.sessions.per.default.queue', llap_concurrency)
+ putHiveInteractiveSitePropertyAttribute('hive.server2.tez.sessions.per.default.queue', "minimum",
+ min_llap_concurrency)
+ putHiveInteractiveSitePropertyAttribute('hive.server2.tez.sessions.per.default.queue', "maximum",
+ LLAP_MAX_CONCURRENCY)
+ Logger.info(
+ "Hive2 config 'hive.server2.tez.sessions.per.default.queue' updated. Min : {0}, Current: {1}, Max: {2}" \
+ .format(min_llap_concurrency, llap_concurrency, LLAP_MAX_CONCURRENCY))
+
+ num_llap_nodes = long(num_llap_nodes)
+
+ putHiveInteractiveEnvProperty('num_llap_nodes', num_llap_nodes)
+ Logger.info("LLAP config 'num_llap_nodes' updated. Current: {0}".format(num_llap_nodes))
+
+ llap_container_size = long(llap_container_size)
+ putHiveInteractiveSiteProperty('hive.llap.daemon.yarn.container.mb', llap_container_size)
+ Logger.info("LLAP config 'hive.llap.daemon.yarn.container.mb' updated. Current: {0}".format(llap_container_size))
+
+ num_executors_per_node = long(num_executors_per_node)
+ putHiveInteractiveSiteProperty('hive.llap.daemon.num.executors', num_executors_per_node)
+ Logger.info("LLAP config 'hive.llap.daemon.num.executors' updated. Current: {0}".format(num_executors_per_node))
+ # 'hive.llap.io.threadpool.size' config value is to be set same as value calculated for
+ # 'hive.llap.daemon.num.executors' at all times.
+ putHiveInteractiveSiteProperty('hive.llap.io.threadpool.size', num_executors_per_node)
+ Logger.info("LLAP config 'hive.llap.io.threadpool.size' updated. Current: {0}".format(num_executors_per_node))
+
+ cache_size_per_node = long(cache_size_per_node)
+ putHiveInteractiveSiteProperty('hive.llap.io.memory.size', cache_size_per_node)
+ Logger.info("LLAP config 'hive.llap.io.memory.size' updated. Current: {0}".format(cache_size_per_node))
+ llap_io_enabled = 'false'
+ if cache_size_per_node >= 64:
+ llap_io_enabled = 'true'
+
+ putHiveInteractiveSiteProperty('hive.llap.io.enabled', llap_io_enabled)
+ Logger.info("Hive2 config 'hive.llap.io.enabled' updated to '{0}' as part of "
+ "'hive.llap.io.memory.size' calculation.".format(llap_io_enabled))
+
+ llap_xmx = long(llap_xmx)
+ putHiveInteractiveEnvProperty('llap_heap_size', llap_xmx)
+ Logger.info("LLAP config 'llap_heap_size' updated. Current: {0}".format(llap_xmx))
+
+ slider_am_container_size = long(slider_am_container_size)
+ putHiveInteractiveEnvProperty('slider_am_container_mb', slider_am_container_size)
+ Logger.info("LLAP config 'slider_am_container_mb' updated. Current: {0}".format(slider_am_container_size))
except Exception as e:
# Set default values, if caught an Exception. The 'llap queue capacity' is left untouched, as it can be increased,
@@ -748,26 +729,18 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
putHiveInteractiveEnvProperty('num_llap_nodes', 0)
putHiveInteractiveEnvPropertyAttribute('num_llap_nodes', "minimum", 1)
putHiveInteractiveEnvPropertyAttribute('num_llap_nodes', "maximum", node_manager_cnt)
- putHiveInteractiveEnvProperty('num_llap_nodes_copy', 0)
- putHiveInteractiveEnvPropertyAttribute('num_llap_nodes_copy', "minimum", 1)
- putHiveInteractiveEnvPropertyAttribute('num_llap_nodes_copy', "maximum", node_manager_cnt)
putHiveInteractiveSiteProperty('hive.llap.daemon.yarn.container.mb', yarn_min_container_size)
putHiveInteractiveSitePropertyAttribute('hive.llap.daemon.yarn.container.mb', "minimum", yarn_min_container_size)
- putHiveInteractiveSiteProperty('hive.llap.daemon.yarn.container.mb_copy', yarn_min_container_size)
- putHiveInteractiveSitePropertyAttribute('hive.llap.daemon.yarn.container.mb_copy', "minimum", yarn_min_container_size)
putHiveInteractiveSiteProperty('hive.llap.daemon.num.executors', 0)
putHiveInteractiveSitePropertyAttribute('hive.llap.daemon.num.executors', "minimum", 1)
- putHiveInteractiveSiteProperty('hive.llap.daemon.num.executors_copy', 0)
- putHiveInteractiveSitePropertyAttribute('hive.llap.daemon.num.executors_copy', "minimum", 1)
putHiveInteractiveSiteProperty('hive.llap.io.threadpool.size', 0)
putHiveInteractiveSiteProperty('hive.llap.io.threadpool.size', 0)
putHiveInteractiveSiteProperty('hive.llap.io.memory.size', 0)
- putHiveInteractiveSiteProperty('hive.llap.io.memory.size_copy', 0)
putHiveInteractiveEnvProperty('llap_heap_size', 0)
http://git-wip-us.apache.org/repos/asf/ambari/blob/b79806ec/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
index e4fad77..558d32d 100644
--- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
@@ -3542,12 +3542,6 @@ class TestHDP25StackAdvisor(TestCase):
self.assertEqual(configurations['tez-interactive-site']['properties']['tez.am.resource.memory.mb'], '512')
- # Check '*_copy' configs, used for displaying it as label on UI.
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb_copy'], '1024')
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors_copy'], '1')
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size_copy'], '512')
- self.assertEqual(configurations['hive-interactive-env']['properties']['num_llap_nodes_copy'], '1')
-
# Test 20: (1). 'default' and 'llap' (State : RUNNING) queue exists at root level in capacity-scheduler, and
@@ -3742,14 +3736,6 @@ class TestHDP25StackAdvisor(TestCase):
self.assertEqual(configurations['tez-interactive-site']['properties']['tez.am.resource.memory.mb'], '682')
- # Check '*_copy' configs, used for displaying it as label on UI.
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb_copy'], '5115')
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors_copy'], '1')
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size_copy'], '4774')
- self.assertEqual(configurations['hive-interactive-env']['properties']['num_llap_nodes_copy'], '1')
-
-
-
@@ -3946,12 +3932,6 @@ class TestHDP25StackAdvisor(TestCase):
self.assertEqual(configurations['tez-interactive-site']['properties']['tez.am.resource.memory.mb'], '1024')
- # Check '*_copy' configs, used for displaying it as label on UI.
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb_copy'], '17408')
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors_copy'], '1')
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size_copy'], '16384')
- self.assertEqual(configurations['hive-interactive-env']['properties']['num_llap_nodes_copy'], '1')
-
####################### 'Three Node Managers' cluster - tests for calculating llap configs ################
@@ -4151,11 +4131,6 @@ class TestHDP25StackAdvisor(TestCase):
self.assertEqual(configurations['tez-interactive-site']['properties']['tez.am.resource.memory.mb'], '2048')
- # Check '*_copy' configs, used for displaying it as label on UI.
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb_copy'], '40960')
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors_copy'], '4')
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size_copy'], '36864')
- self.assertEqual(configurations['hive-interactive-env']['properties']['num_llap_nodes_copy'], '2')
@@ -4351,11 +4326,6 @@ class TestHDP25StackAdvisor(TestCase):
self.assertEqual(configurations['tez-interactive-site']['properties']['tez.am.resource.memory.mb'], '682')
- # Check '*_copy' configs, used for displaying it as label on UI.
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb_copy'], '12276')
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors_copy'], '3')
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size_copy'], '9204')
- self.assertEqual(configurations['hive-interactive-env']['properties']['num_llap_nodes_copy'], '2')
@@ -4549,11 +4519,6 @@ class TestHDP25StackAdvisor(TestCase):
self.assertEqual(configurations['tez-interactive-site']['properties']['tez.am.resource.memory.mb'], '2048')
- # Check '*_copy' configs, used for displaying it as label on UI.
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb_copy'], '204800')
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors_copy'], '3')
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size_copy'], '201728')
- self.assertEqual(configurations['hive-interactive-env']['properties']['num_llap_nodes_copy'], '1')
@@ -4753,11 +4718,6 @@ class TestHDP25StackAdvisor(TestCase):
self.assertEqual(configurations['tez-interactive-site']['properties']['tez.am.resource.memory.mb'], '3072')
- # Check '*_copy' configs, used for displaying it as label on UI.
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb_copy'], '39936')
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors_copy'], '4')
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size_copy'], '35840')
- self.assertEqual(configurations['hive-interactive-env']['properties']['num_llap_nodes_copy'], '3')
@@ -4954,11 +4914,6 @@ class TestHDP25StackAdvisor(TestCase):
self.assertEqual(configurations['tez-interactive-site']['properties']['tez.am.resource.memory.mb'], '1705')
- # Check '*_copy' configs, used for displaying it as label on UI.
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb_copy'], '204600')
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors_copy'], '10')
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size_copy'], '201190')
- self.assertEqual(configurations['hive-interactive-env']['properties']['num_llap_nodes_copy'], '4')
@@ -5148,11 +5103,6 @@ class TestHDP25StackAdvisor(TestCase):
self.assertEqual(configurations['tez-interactive-site']['properties']['tez.am.resource.memory.mb'], '2048')
- # Check '*_copy' configs, used for displaying it as label on UI.
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb_copy'], '204800')
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors_copy'], '3')
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size_copy'], '201728')
- self.assertEqual(configurations['hive-interactive-env']['properties']['num_llap_nodes_copy'], '2')
@@ -5335,11 +5285,6 @@ class TestHDP25StackAdvisor(TestCase):
self.assertEqual(configurations['tez-interactive-site']['properties']['tez.am.resource.memory.mb'], '2048')
- # Check '*_copy' configs, used for displaying it as label on UI.
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb_copy'], '204800')
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors_copy'], '3')
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size_copy'], '201728')
- self.assertEqual(configurations['hive-interactive-env']['properties']['num_llap_nodes_copy'], '2')
@@ -5525,11 +5470,6 @@ class TestHDP25StackAdvisor(TestCase):
self.assertEqual(configurations['tez-interactive-site']['properties']['tez.am.resource.memory.mb'], '2048')
- # Check '*_copy' configs, used for displaying it as label on UI.
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb_copy'], '204800')
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors_copy'], '3')
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size_copy'], '201728')
- self.assertEqual(configurations['hive-interactive-env']['properties']['num_llap_nodes_copy'], '2')
@@ -5607,13 +5547,7 @@ class TestHDP25StackAdvisor(TestCase):
]
}
],
- "changed-configurations": [
- {
- u'old_value': u'3',
- u'type': u'hive-interactive-site',
- u'name': u'hive.server2.tez.sessions.per.default.queue'
- }
- ],
+ "changed-configurations": [ ],
"configurations": {
"capacity-scheduler": {
"properties": {
@@ -5724,29 +5658,24 @@ class TestHDP25StackAdvisor(TestCase):
}
self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, self.hosts)
- self.assertEqual(configurations['hive-interactive-env']['properties']['num_llap_nodes'], '4')
+ self.assertEqual(configurations['hive-interactive-env']['properties']['num_llap_nodes'], '3')
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'], '204800')
+ self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'], '10240')
self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors'], '3')
self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.threadpool.size'], '3')
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size'], '201728')
+ self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size'], '7168')
self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.enabled'], 'true')
self.assertEqual(configurations['hive-interactive-env']['properties']['llap_heap_size'], '2457')
- self.assertEqual(configurations['hive-interactive-env']['properties']['slider_am_container_mb'], '2048')
+ self.assertEqual(configurations['hive-interactive-env']['properties']['slider_am_container_mb'], '512')
self.assertEquals(configurations['hive-interactive-env']['property_attributes']['llap_queue_capacity'],
{'visible': 'false'})
- self.assertEqual(configurations['tez-interactive-site']['properties']['tez.am.resource.memory.mb'], '2048')
+ self.assertEqual(configurations['tez-interactive-site']['properties']['tez.am.resource.memory.mb'], '512')
- # Check '*_copy' configs, used for displaying it as label on UI.
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb_copy'], '204800')
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors_copy'], '3')
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size_copy'], '201728')
- self.assertEqual(configurations['hive-interactive-env']['properties']['num_llap_nodes_copy'], '4')
@@ -5961,11 +5890,6 @@ class TestHDP25StackAdvisor(TestCase):
self.assertTrue('tez.am.resource.memory.mb' not in configurations['tez-interactive-site']['properties'])
- # Check '*_copy' configs, used for displaying it as label on UI.
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb_copy'], '2048')
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors_copy'], '0')
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size_copy'], '0')
- self.assertEqual(configurations['hive-interactive-env']['properties']['num_llap_nodes_copy'], '0')
@@ -6142,11 +6066,6 @@ class TestHDP25StackAdvisor(TestCase):
self.assertEquals(configurations['hive-interactive-env']['property_attributes']['llap_queue_capacity'],
{'maximum': '100', 'minimum': '20', 'visible': 'true'})
- # Check '*_copy' configs, used for displaying it as label on UI.
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb_copy'], '10240')
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors_copy'], '1')
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size_copy'], '512')
- self.assertEqual(configurations['hive-interactive-env']['properties']['num_llap_nodes_copy'], '1')
@@ -6336,11 +6255,6 @@ class TestHDP25StackAdvisor(TestCase):
self.assertEquals(configurations['hive-interactive-env']['property_attributes']['llap_queue_capacity'],
{'visible': 'false'})
- # Check '*_copy' configs, used for displaying it as label on UI.
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb_copy'], '8192')
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors_copy'], '1')
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size_copy'], '0')
- self.assertEqual(configurations['hive-interactive-env']['properties']['num_llap_nodes_copy'], '2')
@@ -6510,11 +6424,6 @@ class TestHDP25StackAdvisor(TestCase):
self.expected_hive_interactive_site_default['hive-interactive-site']['properties']['hive.llap.daemon.queue.name'])
self.assertEquals(configurations['hive-interactive-site']['properties']['hive.server2.tez.default.queues'], 'default')
- # Check '*_copy' configs, used for displaying it as label on UI.
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb_copy'], '4096')
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors_copy'], '1')
- self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size_copy'], '512')
- self.assertEqual(configurations['hive-interactive-env']['properties']['num_llap_nodes_copy'], '1')