You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by nc...@apache.org on 2017/02/13 22:13:36 UTC
[28/50] [abbrv] ambari git commit: AMBARI-19971.
HiveServerInteractive. (1). Use the correct value for Node(s) used for
running LLAP Daemon(s). (2). Set 'num_llap_nodes' calculated value only for
non Ambari managed queue.
AMBARI-19971. HiveServerInteractive. (1). Use the correct value for Node(s) used for running LLAP Daemon(s). (2). Set 'num_llap_nodes' calculated value only for non Ambari managed queue.
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f4c83843
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f4c83843
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f4c83843
Branch: refs/heads/branch-feature-AMBARI-12556
Commit: f4c83843674764918bd50443705af1fce9852a2b
Parents: 38a17a7
Author: Swapan Shridhar <ss...@hortonworks.com>
Authored: Fri Feb 10 13:29:41 2017 -0800
Committer: Swapan Shridhar <ss...@hortonworks.com>
Committed: Fri Feb 10 15:13:48 2017 -0800
----------------------------------------------------------------------
.../package/scripts/hive_server_interactive.py | 12 ++---
.../0.12.0.2.0/package/scripts/params_linux.py | 2 +
.../HIVE/configuration/hive-interactive-env.xml | 4 +-
.../stacks/HDP/2.5/services/stack_advisor.py | 9 +++-
.../HIVE/configuration/hive-interactive-env.xml | 31 ++++++++++++-
.../stacks/2.5/common/test_stack_advisor.py | 49 ++++++++++++++------
6 files changed, 83 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/f4c83843/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py
index 1f1d9a8..2ab48ae 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py
@@ -281,10 +281,9 @@ class HiveServerInteractiveDefault(HiveServerInteractive):
unique_name = "llap-slider%s" % datetime.utcnow().strftime('%Y-%m-%d_%H-%M-%S')
- cmd = format("{stack_root}/current/hive-server2-hive2/bin/hive --service llap --instances {params.num_llap_nodes}"
- " --slider-am-container-mb {params.slider_am_container_mb} --size {params.llap_daemon_container_size}m"
- " --cache {params.hive_llap_io_mem_size}m --xmx {params.llap_heap_size}m --loglevel {params.llap_log_level}"
- " {params.llap_extra_slider_opts} --output {LLAP_PACKAGE_CREATION_PATH}/{unique_name}")
+ cmd = format("{stack_root}/current/hive-server2-hive2/bin/hive --service llap --slider-am-container-mb {params.slider_am_container_mb} "
+ "--size {params.llap_daemon_container_size}m --cache {params.hive_llap_io_mem_size}m --xmx {params.llap_heap_size}m "
+ "--loglevel {params.llap_log_level} {params.llap_extra_slider_opts} --output {LLAP_PACKAGE_CREATION_PATH}/{unique_name}")
# Append params that are supported from Hive llap GA version.
if params.stack_supports_hive_interactive_ga:
@@ -300,11 +299,12 @@ class HiveServerInteractiveDefault(HiveServerInteractive):
else:
Logger.info("Setting slider_placement: 4, as llap_daemon_container_size : {0} <= 0.5 * "
"YARN NodeManager Memory({1})".format(params.llap_daemon_container_size, params.yarn_nm_mem))
- cmd += format(" --slider-placement {slider_placement} --skiphadoopversion --skiphbasecp")
+ cmd += format(" --slider-placement {slider_placement} --skiphadoopversion --skiphbasecp --instances {params.num_llap_daemon_running_nodes}")
# Setup the logger for the ga version only
cmd += format(" --logger {params.llap_logger}")
-
+ else:
+ cmd += format(" --instances {params.num_llap_nodes}")
if params.security_enabled:
llap_keytab_splits = params.hive_llap_keytab_file.split("/")
Logger.debug("llap_keytab_splits : {0}".format(llap_keytab_splits))
http://git-wip-us.apache.org/repos/asf/ambari/blob/f4c83843/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
index a5618ca..936b194 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
@@ -685,6 +685,8 @@ if has_hive_interactive:
num_retries_for_checking_llap_status = default('/configurations/hive-interactive-env/num_retries_for_checking_llap_status', 10)
# Used in LLAP slider package creation
yarn_nm_mem = config['configurations']['yarn-site']['yarn.nodemanager.resource.memory-mb']
+ if stack_supports_hive_interactive_ga:
+ num_llap_daemon_running_nodes = config['configurations']['hive-interactive-env']['num_llap_nodes_for_llap_daemons']
num_llap_nodes = config['configurations']['hive-interactive-env']['num_llap_nodes']
llap_daemon_container_size = config['configurations']['hive-interactive-site']['hive.llap.daemon.yarn.container.mb']
llap_log_level = config['configurations']['hive-interactive-env']['llap_log_level']
http://git-wip-us.apache.org/repos/asf/ambari/blob/f4c83843/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml
index e636e0c..ef9be76 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml
@@ -45,8 +45,8 @@
<property>
<name>num_llap_nodes</name>
<value>1</value>
- <description>The number of Hive LLAP daemons to run.</description>
- <display-name>Number of LLAP Nodes</display-name>
+ <description>Number of nodes used by Hive's LLAP, which includes nodes running : LLAP daemon, Slider and Tez App Master(s).</description>
+ <display-name>Number of nodes used by Hive's LLAP</display-name>
<value-attributes>
<type>int</type>
<minimum>1</minimum>
http://git-wip-us.apache.org/repos/asf/ambari/blob/f4c83843/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
index 99b6776..9fe8fc3 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
@@ -951,10 +951,13 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
Logger.info("DBG: Calculated '{0}' queue available capacity : {1}, using following: llap_daemon_selected_queue_cap : {2}, "
"yarn_min_container_size : {3}".format(llap_daemon_selected_queue_name, total_llap_mem_normalized,
llap_daemon_selected_queue_cap, yarn_min_container_size))
- '''Rounding up numNodes so that we run more daemons, and utilitze more CPUs. The rest of the calcaulkations will take care of cutting this down if required'''
+ '''Rounding up numNodes so that we run more daemons, and utilitze more CPUs. The rest of the calcaulations will take care of cutting this down if required'''
num_llap_nodes_requested = math.ceil(total_llap_mem_normalized / yarn_nm_mem_in_mb_normalized)
Logger.info("DBG: Calculated 'num_llap_nodes_requested' : {0}, using following: total_llap_mem_normalized : {1}, "
"yarn_nm_mem_in_mb_normalized : {2}".format(num_llap_nodes_requested, total_llap_mem_normalized, yarn_nm_mem_in_mb_normalized))
+ # Pouplate the 'num_llap_nodes_requested' in config 'num_llap_nodes', a read only config for non-Ambari managed queue case.
+ putHiveInteractiveEnvProperty('num_llap_nodes', num_llap_nodes_requested)
+ Logger.info("Setting config 'num_llap_nodes' as : {0}".format(num_llap_nodes_requested))
queue_am_fraction_perc = float(self.__getQueueAmFractionFromCapacityScheduler(capacity_scheduler_properties, llap_daemon_selected_queue_name))
hive_tez_am_cap_available = queue_am_fraction_perc * total_llap_mem_normalized
Logger.info("DBG: Calculated 'hive_tez_am_cap_available' : {0}, using following: queue_am_fraction_perc : {1}, "
@@ -1182,7 +1185,8 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
Logger.info("User requested num_llap_nodes : {0}, but used/adjusted value for calculations is : {1}".format(num_llap_nodes_requested, num_llap_nodes))
else:
Logger.info("Used num_llap_nodes for calculations : {0}".format(num_llap_nodes_requested))
- putHiveInteractiveEnvProperty('num_llap_nodes', num_llap_nodes)
+ putHiveInteractiveEnvProperty('num_llap_nodes_for_llap_daemons', num_llap_nodes)
+ Logger.info("Setting config 'num_llap_nodes_for_llap_daemons' as : {0}".format(num_llap_nodes))
llap_container_size = long(llap_daemon_mem_per_node)
putHiveInteractiveSiteProperty('hive.llap.daemon.yarn.container.mb', llap_container_size)
@@ -1243,6 +1247,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
putHiveInteractiveSitePropertyAttribute('hive.server2.tez.sessions.per.default.queue', "minimum", 1)
putHiveInteractiveSitePropertyAttribute('hive.server2.tez.sessions.per.default.queue', "maximum", 1)
putHiveInteractiveEnvProperty('num_llap_nodes', 0)
+ putHiveInteractiveEnvProperty('num_llap_nodes_for_llap_daemons', 0)
putHiveInteractiveEnvPropertyAttribute('num_llap_nodes', "minimum", 1)
putHiveInteractiveEnvPropertyAttribute('num_llap_nodes', "maximum", node_manager_cnt)
putHiveInteractiveSiteProperty('hive.llap.daemon.yarn.container.mb', yarn_min_container_size)
http://git-wip-us.apache.org/repos/asf/ambari/blob/f4c83843/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-env.xml
index af656f4..b659205 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-env.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-env.xml
@@ -62,7 +62,36 @@
</value-attributes>
<on-ambari-upgrade add="false"/>
</property>
-
+ <property>
+ <name>num_llap_nodes_for_llap_daemons</name>
+ <value>1</value>
+ <description>Number of Node(s) on which Hive LLAP daemon runs.</description>
+ <display-name>Number of Node(s) for running Hive LLAP daemon</display-name>
+ <value-attributes>
+ <type>int</type>
+ <minimum>1</minimum>
+ <increment-step>1</increment-step>
+ </value-attributes>
+ <depends-on>
+ <property>
+ <type>hive-interactive-env</type>
+ <name>num_llap_nodes</name>
+ </property>
+ <property>
+ <type>hive-interactive-env</type>
+ <name>enable_hive_interactive</name>
+ </property>
+ <property>
+ <type>hive-interactive-site</type>
+ <name>hive.llap.daemon.queue.name</name>
+ </property>
+ <property>
+ <type>capacity-scheduler</type>
+ <name>yarn.scheduler.capacity.root.queues</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="true"/>
+ </property>
<!-- hive-env.sh -->
http://git-wip-us.apache.org/repos/asf/ambari/blob/f4c83843/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
index ec56bad..ef83bca 100644
--- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
@@ -1169,8 +1169,8 @@ class TestHDP25StackAdvisor(TestCase):
self.assertEqual(configurations['hive-interactive-site']['properties']['hive.server2.tez.sessions.per.default.queue'], '1')
self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.server2.tez.sessions.per.default.queue'], {'minimum': '1', 'maximum': '4'})
- self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes'], 3)
-
+ self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes_for_llap_daemons'], 3)
+ self.assertTrue('num_llap_nodes' not in configurations['hive-interactive-env']['properties'])
self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'], '9216')
@@ -1369,6 +1369,8 @@ class TestHDP25StackAdvisor(TestCase):
self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.server2.tez.sessions.per.default.queue'], {'maximum': '3.0'})
self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes'], 3)
+ self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes_for_llap_daemons'], 3)
+
self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'], '9548')
self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors'], '1')
@@ -1561,7 +1563,9 @@ class TestHDP25StackAdvisor(TestCase):
self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.server2.tez.sessions.per.default.queue'], {'maximum': '4'})
- self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes'], 3)
+ self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes_for_llap_daemons'], 3)
+ self.assertTrue('num_llap_nodes' not in configurations['hive-interactive-env']['properties'])
+
self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'], '48128')
self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors'], '1')
@@ -1762,7 +1766,9 @@ class TestHDP25StackAdvisor(TestCase):
self.assertEqual(configurations['hive-interactive-site']['properties']['hive.server2.tez.sessions.per.default.queue'], '1')
self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.server2.tez.sessions.per.default.queue'], {'maximum': '4', 'minimum': '1'})
- self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes'], 3)
+ self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes_for_llap_daemons'], 3)
+ self.assertTrue('num_llap_nodes' not in configurations['hive-interactive-env']['properties'])
+
self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'], '38912')
self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors'], '4')
@@ -1953,14 +1959,15 @@ class TestHDP25StackAdvisor(TestCase):
configurations = {
}
-
self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, self.hosts)
-
self.assertEqual(configurations['hive-interactive-site']['properties']['hive.server2.tez.sessions.per.default.queue'], '1')
self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.server2.tez.sessions.per.default.queue'], {'minimum': '1', 'maximum': '4'})
self.assertEqual(configurations['capacity-scheduler']['properties'], {'capacity-scheduler': 'yarn.scheduler.capacity.root.accessible-node-labels=*\nyarn.scheduler.capacity.maximum-am-resource-percent=1\nyarn.scheduler.capacity.node-locality-delay=40\nyarn.scheduler.capacity.root.capacity=100\nyarn.scheduler.capacity.root.default.state=RUNNING\nyarn.scheduler.capacity.root.default.maximum-capacity=66.0\nyarn.scheduler.capacity.root.queues=default,llap\nyarn.scheduler.capacity.maximum-applications=10000\nyarn.scheduler.capacity.root.default.user-limit-factor=1\nyarn.scheduler.capacity.root.acl_administer_queue=*\nyarn.scheduler.capacity.root.default.acl_submit_applications=*\nyarn.scheduler.capacity.root.default.capacity=66.0\nyarn.scheduler.capacity.queue-mappings-override.enable=false\nyarn.scheduler.capacity.root.llap.user-limit-factor=1\nyarn.scheduler.capacity.root.llap.state=RUNNING\nyarn.scheduler.capacity.root.llap.ordering-policy=fifo\nyarn.scheduler.capacity.root.llap.mi
nimum-user-limit-percent=100\nyarn.scheduler.capacity.root.llap.maximum-capacity=34.0\nyarn.scheduler.capacity.root.llap.capacity=34.0\nyarn.scheduler.capacity.root.llap.acl_submit_applications=hive\nyarn.scheduler.capacity.root.llap.acl_administer_queue=hive\nyarn.scheduler.capacity.root.llap.maximum-am-resource-percent=1'})
- self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes'], 3)
+ self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes_for_llap_daemons'], 3)
+ self.assertTrue('num_llap_nodes' not in configurations['hive-interactive-env']['properties'])
+
+ self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes_for_llap_daemons'], 3)
self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'], '10571')
self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors'], '3')
@@ -1988,7 +1995,7 @@ class TestHDP25StackAdvisor(TestCase):
# Small configuration test with 3 nodes - 'yarn.nodemanager.resource.memory-mb' : 2046 and 'yarn.scheduler.minimum-allocation-mb' : 682, representing a small GCE cluster.
#
# Expected : Configurations values recommended for llap related configs.
- def test_recommendYARNConfigurations_three_node_manager_llap_configs_updated_2(self):
+ def test_recommendYARNConfigurations_three_node_manager_llap_configs_updated_3(self):
# 3 node managers and yarn.nodemanager.resource.memory-mb": "12288"
services = {
"services": [{
@@ -2158,7 +2165,9 @@ class TestHDP25StackAdvisor(TestCase):
self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.server2.tez.sessions.per.default.queue'], {'minimum': '1', 'maximum': '3.0'})
self.assertEqual(configurations['capacity-scheduler']['properties'], {'capacity-scheduler': 'yarn.scheduler.capacity.root.accessible-node-labels=*\nyarn.scheduler.capacity.maximum-am-resource-percent=1\nyarn.scheduler.capacity.node-locality-delay=40\nyarn.scheduler.capacity.root.capacity=100\nyarn.scheduler.capacity.root.default.state=RUNNING\nyarn.scheduler.capacity.root.default.maximum-capacity=66.0\nyarn.scheduler.capacity.root.queues=default,llap\nyarn.scheduler.capacity.maximum-applications=10000\nyarn.scheduler.capacity.root.default.user-limit-factor=1\nyarn.scheduler.capacity.root.acl_administer_queue=*\nyarn.scheduler.capacity.root.default.acl_submit_applications=*\nyarn.scheduler.capacity.root.default.capacity=66.0\nyarn.scheduler.capacity.queue-mappings-override.enable=false\nyarn.scheduler.capacity.root.llap.user-limit-factor=1\nyarn.scheduler.capacity.root.llap.state=RUNNING\nyarn.scheduler.capacity.root.llap.ordering-policy=fifo\nyarn.scheduler.capacity.root.llap.mi
nimum-user-limit-percent=100\nyarn.scheduler.capacity.root.llap.maximum-capacity=34.0\nyarn.scheduler.capacity.root.llap.capacity=34.0\nyarn.scheduler.capacity.root.llap.acl_submit_applications=hive\nyarn.scheduler.capacity.root.llap.acl_administer_queue=hive\nyarn.scheduler.capacity.root.llap.maximum-am-resource-percent=1'})
- self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes'], 3)
+ self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes_for_llap_daemons'], 3)
+ self.assertTrue('num_llap_nodes' not in configurations['hive-interactive-env']['properties'])
+
self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'], '682')
self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors'], '1')
@@ -2354,7 +2363,8 @@ class TestHDP25StackAdvisor(TestCase):
self.assertEqual(configurations['capacity-scheduler']['properties'], {'capacity-scheduler': 'yarn.scheduler.capacity.root.accessible-node-labels=*\nyarn.scheduler.capacity.maximum-am-resource-percent=1\nyarn.scheduler.capacity.node-locality-delay=40\nyarn.scheduler.capacity.root.capacity=100\nyarn.scheduler.capacity.root.default.state=RUNNING\nyarn.scheduler.capacity.root.default.maximum-capacity=0.0\nyarn.scheduler.capacity.root.queues=default,llap\nyarn.scheduler.capacity.maximum-applications=10000\nyarn.scheduler.capacity.root.default.user-limit-factor=1\nyarn.scheduler.capacity.root.acl_administer_queue=*\nyarn.scheduler.capacity.root.default.acl_submit_applications=*\nyarn.scheduler.capacity.root.default.capacity=0.0\nyarn.scheduler.capacity.queue-mappings-override.enable=false\nyarn.scheduler.capacity.root.llap.user-limit-factor=1\nyarn.scheduler.capacity.root.llap.state=RUNNING\nyarn.scheduler.capacity.root.llap.ordering-policy=fifo\nyarn.scheduler.capacity.root.llap.mini
mum-user-limit-percent=100\nyarn.scheduler.capacity.root.llap.maximum-capacity=100.0\nyarn.scheduler.capacity.root.llap.capacity=100.0\nyarn.scheduler.capacity.root.llap.acl_submit_applications=hive\nyarn.scheduler.capacity.root.llap.acl_administer_queue=hive\nyarn.scheduler.capacity.root.llap.maximum-am-resource-percent=1'})
self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.server2.tez.sessions.per.default.queue'], {'maximum': '4'})
- self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes'], 3)
+ self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes_for_llap_daemons'], 3)
+ self.assertTrue('num_llap_nodes' not in configurations['hive-interactive-env']['properties'])
self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'], '202752')
self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors'], '3')
@@ -2557,7 +2567,9 @@ class TestHDP25StackAdvisor(TestCase):
self.assertEqual(configurations['hive-interactive-site']['properties']['hive.server2.tez.sessions.per.default.queue'], '1.0')
self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.server2.tez.sessions.per.default.queue'], {'maximum': '4', 'minimum': '1'})
- self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes'], 5)
+ self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes_for_llap_daemons'], 3)
+ self.assertTrue('num_llap_nodes' not in configurations['hive-interactive-env']['properties'])
+
self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'], '36864')
self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors'], '4')
@@ -2754,7 +2766,9 @@ class TestHDP25StackAdvisor(TestCase):
self.assertEqual(configurations['hive-interactive-site']['properties']['hive.server2.tez.sessions.per.default.queue'], '1')
self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.server2.tez.sessions.per.default.queue'], {'maximum': '4', 'minimum': '1'})
- self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes'], 3)
+ self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes_for_llap_daemons'], 3)
+ self.assertTrue('num_llap_nodes' not in configurations['hive-interactive-env']['properties'])
+
self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'], '203918')
self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors'], '10')
@@ -2947,7 +2961,9 @@ class TestHDP25StackAdvisor(TestCase):
self.assertEqual(configurations['capacity-scheduler']['properties'], {'capacity-scheduler': 'yarn.scheduler.capacity.root.accessible-node-labels=*\nyarn.scheduler.capacity.maximum-am-resource-percent=1\nyarn.scheduler.capacity.node-locality-delay=40\nyarn.scheduler.capacity.root.capacity=100\nyarn.scheduler.capacity.root.default.state=RUNNING\nyarn.scheduler.capacity.root.default.maximum-capacity=0.0\nyarn.scheduler.capacity.root.queues=default,llap\nyarn.scheduler.capacity.maximum-applications=10000\nyarn.scheduler.capacity.root.default.user-limit-factor=1\nyarn.scheduler.capacity.root.acl_administer_queue=*\nyarn.scheduler.capacity.root.default.acl_submit_applications=*\nyarn.scheduler.capacity.root.default.capacity=0.0\nyarn.scheduler.capacity.queue-mappings-override.enable=false\nyarn.scheduler.capacity.root.llap.user-limit-factor=1\nyarn.scheduler.capacity.root.llap.state=RUNNING\nyarn.scheduler.capacity.root.llap.ordering-policy=fifo\nyarn.scheduler.capacity.root.llap.mini
mum-user-limit-percent=100\nyarn.scheduler.capacity.root.llap.maximum-capacity=100.0\nyarn.scheduler.capacity.root.llap.capacity=100.0\nyarn.scheduler.capacity.root.llap.acl_submit_applications=hive\nyarn.scheduler.capacity.root.llap.acl_administer_queue=hive\nyarn.scheduler.capacity.root.llap.maximum-am-resource-percent=1'})
self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.server2.tez.sessions.per.default.queue'], {'maximum': '4'})
- self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes'], 3)
+ self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes_for_llap_daemons'], 3)
+ self.assertTrue('num_llap_nodes' not in configurations['hive-interactive-env']['properties'])
+
self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'], '202752')
self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors'], '3')
@@ -3139,6 +3155,8 @@ class TestHDP25StackAdvisor(TestCase):
self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.server2.tez.sessions.per.default.queue'], {'maximum': '4'})
self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes'], 3)
+ self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes_for_llap_daemons'], 3)
+
self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'], '202752')
self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors'], '3')
@@ -3361,6 +3379,8 @@ class TestHDP25StackAdvisor(TestCase):
self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.server2.tez.sessions.per.default.queue'], {'maximum': '4', 'minimum': '1'})
self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes'], 3)
+ self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes_for_llap_daemons'], 3)
+
self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'], '204288')
self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors'], '3')
@@ -3580,6 +3600,7 @@ class TestHDP25StackAdvisor(TestCase):
self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, self.hosts)
self.assertEqual(configurations['hive-interactive-env']['properties']['num_llap_nodes'], '0')
+ self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes_for_llap_daemons'], 0)
self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'], '2048')
@@ -3962,6 +3983,8 @@ class TestHDP25StackAdvisor(TestCase):
self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.server2.tez.sessions.per.default.queue'], {'maximum': '4', 'minimum': '1'})
self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes'], 1)
+ self.assertTrue(configurations['hive-interactive-env']['properties']['num_llap_nodes_for_llap_daemons'], 1)
+
self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'], '200704')
self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors'], '3')