You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sm...@apache.org on 2016/04/29 14:14:20 UTC
ambari git commit: AMBARI-16178. llap queue minimum recommended size
is not persistent across HSI on and off (Swapan Sridhar via smohanty)
Repository: ambari
Updated Branches:
refs/heads/trunk a3c4d46c7 -> 051f09a40
AMBARI-16178. llap queue minimum recommended size is not persistent across HSI on and off (Swapan Sridhar via smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/051f09a4
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/051f09a4
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/051f09a4
Branch: refs/heads/trunk
Commit: 051f09a400509fe1e38b768553a6bcd755e4ed5e
Parents: a3c4d46
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Fri Apr 29 05:14:06 2016 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Fri Apr 29 05:14:06 2016 -0700
----------------------------------------------------------------------
.../stacks/HDP/2.5/services/stack_advisor.py | 22 +++---
.../stacks/2.5/common/test_stack_advisor.py | 76 +++++++++++++-------
2 files changed, 62 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/051f09a4/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
index 03af7c0..ca92157 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
@@ -506,7 +506,8 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
node_manager_hosts = self.get_node_manager_hosts(services, hosts)
node_manager_host_cnt = len(node_manager_hosts)
- num_llap_nodes_in_changed_configs = self.are_config_props_in_changed_configs(services, "hive-interactive-env", "num_llap_nodes", False)
+ num_llap_nodes_in_changed_configs = self.are_config_props_in_changed_configs(services, "hive-interactive-env",
+ set(['num_llap_nodes']), False)
if not num_llap_nodes_in_changed_configs:
num_llap_nodes, num_llap_nodes_max_limit = self.calculate_num_llap_nodes(services, hosts, configurations)
else:
@@ -736,12 +737,11 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
if 'llap_queue_capacity' in services['configurations']['hive-interactive-env']['properties']:
llap_slider_cap_percentage = int(
services['configurations']['hive-interactive-env']['properties']['llap_queue_capacity'])
+ llap_min_reqd_cap_percentage = self.min_llap_queue_perc_required_in_cluster(services, hosts)
if llap_slider_cap_percentage not in range(1,101):
- llap_slider_cap_percentage = self.min_llap_queue_perc_required_in_cluster(services, hosts)
- Logger.info("Adjusting HIVE 'llap_queue_capacity' from {0}% to {1}%".format(llap_slider_cap_percentage, llap_slider_cap_percentage))
-
- putHiveInteractiveEnvProperty('llap_queue_capacity', llap_slider_cap_percentage)
- putHiveInteractiveEnvPropertyAttribute('llap_queue_capacity', "minimum", llap_slider_cap_percentage)
+ Logger.info("Adjusting HIVE 'llap_queue_capacity' from {0}% to {1}%".format(llap_slider_cap_percentage, llap_min_reqd_cap_percentage))
+ putHiveInteractiveEnvProperty('llap_queue_capacity', llap_min_reqd_cap_percentage)
+ llap_slider_cap_percentage = llap_min_reqd_cap_percentage
else:
Logger.error("Problem retrieving LLAP Queue Capacity. Skipping creating {0} queue".format(llap_queue_name))
return
@@ -749,8 +749,8 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
capSchedConfigKeys = capacitySchedulerProperties.keys()
yarn_default_queue_capacity = -1
- if 'yarn.scheduler.capacity.root.capacity' in capSchedConfigKeys:
- yarn_default_queue_capacity = capacitySchedulerProperties.get('yarn.scheduler.capacity.root.capacity')
+ if 'yarn.scheduler.capacity.root.default.capacity' in capSchedConfigKeys:
+ yarn_default_queue_capacity = capacitySchedulerProperties.get('yarn.scheduler.capacity.root.default.capacity')
# Get 'llap' queue state
currLlapQueueState = ''
@@ -760,7 +760,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
# Get 'llap' queue capacity
currLlapQueueCap = -1
if 'yarn.scheduler.capacity.root.'+llap_queue_name+'.capacity' in capSchedConfigKeys:
- currLlapQueueCap = capacitySchedulerProperties.get('yarn.scheduler.capacity.root.'+llap_queue_name+'.capacity')
+ currLlapQueueCap = int(capacitySchedulerProperties.get('yarn.scheduler.capacity.root.'+llap_queue_name+'.capacity'))
if self.HIVE_INTERACTIVE_SITE in services['configurations'] and \
'hive.llap.daemon.queue.name' in services['configurations'][self.HIVE_INTERACTIVE_SITE]['properties']:
@@ -784,7 +784,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
((len(leafQueueNames) == 1 and int(yarn_default_queue_capacity) == 100) or \
((len(leafQueueNames) == 2 and llap_queue_name in leafQueueNames) and \
((currLlapQueueState == 'STOPPED' and enabled_hive_int_in_changed_configs) or (currLlapQueueState == 'RUNNING' and currLlapQueueCap != llap_slider_cap_percentage)))):
- adjusted_default_queue_cap = str(int(yarn_default_queue_capacity) - llap_slider_cap_percentage)
+ adjusted_default_queue_cap = str(100 - llap_slider_cap_percentage)
for prop, val in capacitySchedulerProperties.items():
if llap_queue_name not in prop:
if prop == 'yarn.scheduler.capacity.root.queues':
@@ -831,6 +831,8 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
# Update Hive 'hive.llap.daemon.queue.name' prop to use 'llap' queue.
putHiveInteractiveSiteProperty('hive.llap.daemon.queue.name', 'llap')
+ putHiveInteractiveEnvPropertyAttribute('llap_queue_capacity', "minimum", llap_min_reqd_cap_percentage)
+
# Update 'hive.llap.daemon.queue.name' prop combo entries.
self.setLlapDaemonQueueName(services, configurations)
else:
http://git-wip-us.apache.org/repos/asf/ambari/blob/051f09a4/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
index 8f4a5c2..574964f 100644
--- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
@@ -553,7 +553,8 @@ class TestHDP25StackAdvisor(TestCase):
"hive-interactive-site":
{
'properties': {
- 'hive.llap.daemon.queue.name':'default'
+ 'hive.llap.daemon.queue.name':'default',
+ 'hive.tez.container.size': '341'
}
},
"tez-interactive-site": {
@@ -1437,9 +1438,22 @@ class TestHDP25StackAdvisor(TestCase):
"hive-interactive-site":
{
'properties': {
- 'hive.llap.daemon.queue.name':'llap'
+ 'hive.llap.daemon.queue.name':'llap',
+ 'hive.tez.container.size': '341'
}
},
+ "tez-interactive-site": {
+ "properties": {
+ "tez.am.resource.memory.mb": "341"
+ }
+ },
+ "yarn-site": {
+ "properties": {
+ "yarn.scheduler.minimum-allocation-mb": "341",
+ "yarn.nodemanager.resource.memory-mb": "20000",
+ "yarn.nodemanager.resource.cpu-vcores": '1'
+ }
+ },
"hive-env":
{
'properties': {
@@ -1690,7 +1704,7 @@ class TestHDP25StackAdvisor(TestCase):
{
'properties': {
'enable_hive_interactive': 'true',
- 'llap_queue_capacity':'40'
+ 'llap_queue_capacity':'41'
}
},
"hive-interactive-site":
@@ -2144,7 +2158,7 @@ class TestHDP25StackAdvisor(TestCase):
{
'properties': {
'enable_hive_interactive': 'true',
- 'llap_queue_capacity':'40'
+ 'llap_queue_capacity':'41'
}
},
"hive-interactive-site":
@@ -2564,7 +2578,7 @@ class TestHDP25StackAdvisor(TestCase):
{
'properties': {
'enable_hive_interactive': 'true',
- 'llap_queue_capacity':'40',
+ 'llap_queue_capacity':'41',
'num_llap_nodes': 1
}
},
@@ -2930,6 +2944,8 @@ class TestHDP25StackAdvisor(TestCase):
# expected vals.
expected_visibility_false = {'visible': 'false'}
expected_visibility_true = {'visible': 'true'}
+ expected_llap_queue_attributes_min_25_visible_true = {'minimum': '25', 'visible': 'true'}
+ expected_llap_queue_attributes_min_21_visible_true = {'minimum': '21', 'visible': 'true'}
@@ -2943,6 +2959,7 @@ class TestHDP25StackAdvisor(TestCase):
}
self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services_1, hosts)
# Check output
+ print "\n\n\n\n conf = ",configurations
self.assertEquals(configurations['hive-interactive-site']['properties']['hive.llap.daemon.queue.name'],
expected_hive_interactive_site_llap['hive-interactive-site']['properties']['hive.llap.daemon.queue.name'])
self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.daemon.queue.name'],
@@ -3167,29 +3184,32 @@ class TestHDP25StackAdvisor(TestCase):
}
# expected vals.
- yarn_cont_mb_visibility = {'minimum': '682', 'maximum': '682'}
- llap_io_mem_size_visibility = {'minimum': '0', 'maximum': '341'}
- llap_num_executors_visibility = {'minimum': '1', 'maximum': '1'}
- num_llap_nodes_visibility = {'minimum': '1', 'maximum': '1'}
+ print "\n\n\n\n\n\n\n\n\n\n\n"
+ yarn_cont_mb_attributes = {'minimum': '682', 'maximum': '682'}
+ llap_io_mem_size_attributes = {'minimum': '0', 'maximum': '341'}
+ llap_num_executors_attributes = {'minimum': '1', 'maximum': '1'}
+ num_llap_nodes_attributes = {'minimum': '1', 'maximum': '1'}
+
self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services_13, hosts)
self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'], '682')
- self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.daemon.yarn.container.mb'], yarn_cont_mb_visibility)
+ self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.daemon.yarn.container.mb'], yarn_cont_mb_attributes)
self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors'], '1')
- self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.daemon.num.executors'], llap_num_executors_visibility)
+ self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.daemon.num.executors'], llap_num_executors_attributes)
self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size'], '341')
- self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.io.memory.size'], llap_io_mem_size_visibility)
+ self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.io.memory.size'], llap_io_mem_size_attributes)
self.assertEqual(configurations['hive-interactive-env']['properties']['num_llap_nodes'], '1')
- self.assertEquals(configurations['hive-interactive-env']['property_attributes']['num_llap_nodes'], num_llap_nodes_visibility)
+ self.assertEquals(configurations['hive-interactive-env']['property_attributes']['num_llap_nodes'], num_llap_nodes_attributes)
self.assertEqual(configurations['hive-interactive-env']['properties']['llap_heap_size'], '272')
self.assertEqual(configurations['hive-interactive-env']['properties']['slider_am_container_mb'], '341')
- self.assertEquals(configurations['hive-interactive-env']['property_attributes']['llap_queue_capacity'], expected_visibility_true)
+ self.assertEquals(configurations['hive-interactive-env']['property_attributes']['llap_queue_capacity'],
+ expected_llap_queue_attributes_min_25_visible_true)
@@ -3200,10 +3220,10 @@ class TestHDP25StackAdvisor(TestCase):
}
# expected vals.
- yarn_cont_mb_visibility = {'minimum': '682', 'maximum': '682'}
- llap_io_mem_size_visibility = {'minimum': '0', 'maximum': '341'}
- llap_num_executors_visibility = {'minimum': '1', 'maximum': '1'}
- num_llap_nodes_visibility = {'minimum': '1', 'maximum': '1'}
+ yarn_cont_mb_attributes = {'minimum': '682', 'maximum': '682'}
+ llap_io_mem_size_attributes = {'minimum': '0', 'maximum': '341'}
+ llap_num_executors_attributes = {'minimum': '1', 'maximum': '1'}
+ num_llap_nodes_attributes = {'minimum': '1', 'maximum': '1'}
@@ -3211,21 +3231,22 @@ class TestHDP25StackAdvisor(TestCase):
self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.yarn.container.mb'], '682')
- self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.daemon.yarn.container.mb'], yarn_cont_mb_visibility)
+ self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.daemon.yarn.container.mb'], yarn_cont_mb_attributes)
self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.daemon.num.executors'], '1')
- self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.daemon.num.executors'], llap_num_executors_visibility)
+ self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.daemon.num.executors'], llap_num_executors_attributes)
self.assertEqual(configurations['hive-interactive-site']['properties']['hive.llap.io.memory.size'], '341')
- self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.io.memory.size'], llap_io_mem_size_visibility)
+ self.assertEquals(configurations['hive-interactive-site']['property_attributes']['hive.llap.io.memory.size'], llap_io_mem_size_attributes)
self.assertEqual(configurations['hive-interactive-env']['properties']['num_llap_nodes'], '1')
- self.assertEquals(configurations['hive-interactive-env']['property_attributes']['num_llap_nodes'], num_llap_nodes_visibility)
+ self.assertEquals(configurations['hive-interactive-env']['property_attributes']['num_llap_nodes'], num_llap_nodes_attributes)
self.assertEqual(configurations['hive-interactive-env']['properties']['llap_heap_size'], '272')
self.assertEqual(configurations['hive-interactive-env']['properties']['slider_am_container_mb'], '341')
- self.assertEquals(configurations['hive-interactive-env']['property_attributes']['llap_queue_capacity'], expected_visibility_true)
+ self.assertEquals(configurations['hive-interactive-env']['property_attributes']['llap_queue_capacity'],
+ expected_llap_queue_attributes_min_25_visible_true)
@@ -3294,7 +3315,8 @@ class TestHDP25StackAdvisor(TestCase):
self.assertEqual(configurations['hive-interactive-env']['properties']['slider_am_container_mb'], '341')
- self.assertEquals(configurations['hive-interactive-env']['property_attributes']['llap_queue_capacity'], expected_visibility_true)
+ self.assertEquals(configurations['hive-interactive-env']['property_attributes']['llap_queue_capacity'],
+ expected_llap_queue_attributes_min_25_visible_true)
@@ -3332,7 +3354,8 @@ class TestHDP25StackAdvisor(TestCase):
self.assertEqual(configurations['hive-interactive-env']['properties']['slider_am_container_mb'], '682')
- self.assertEquals(configurations['hive-interactive-env']['property_attributes']['llap_queue_capacity'], expected_visibility_true)
+ self.assertEquals(configurations['hive-interactive-env']['property_attributes']['llap_queue_capacity'],
+ expected_llap_queue_attributes_min_21_visible_true)
@@ -3353,7 +3376,8 @@ class TestHDP25StackAdvisor(TestCase):
expected_hive_interactive_site_empty['hive-interactive-site']['properties'])
self.assertEquals(configurations['hive-interactive-env']['properties'],
expected_hive_interactive_env_empty['hive-interactive-env']['properties'])
- self.assertEquals(configurations['hive-interactive-env']['property_attributes']['llap_queue_capacity'], expected_visibility_true)
+ self.assertEquals(configurations['hive-interactive-env']['property_attributes']['llap_queue_capacity'],
+ expected_visibility_true)