You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sr...@apache.org on 2015/06/16 21:52:20 UTC
ambari git commit: AMBARI-11942. YARN Queues not showing up in Hive
configs (srimanth)
Repository: ambari
Updated Branches:
refs/heads/branch-2.1 c3bec2242 -> a03011c7d
AMBARI-11942. YARN Queues not showing up in Hive configs (srimanth)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a03011c7
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a03011c7
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a03011c7
Branch: refs/heads/branch-2.1
Commit: a03011c7d2eab80e9c3543928eead3f715fc35ca
Parents: c3bec22
Author: Srimanth Gunturi <sg...@hortonworks.com>
Authored: Tue Jun 16 01:51:49 2015 -0700
Committer: Srimanth Gunturi <sg...@hortonworks.com>
Committed: Tue Jun 16 12:52:16 2015 -0700
----------------------------------------------------------------------
.../services/HIVE/configuration/hive-site.xml | 6 ++
.../stacks/HDP/2.2/services/stack_advisor.py | 28 +++++++---
.../stacks/2.2/common/test_stack_advisor.py | 59 ++++++++++++++++++++
3 files changed, 84 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/a03011c7/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml
index 9c53773..6f9c22b 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml
@@ -1592,6 +1592,12 @@ limitations under the License.
</entries>
<selection-cardinality>1+</selection-cardinality>
</value-attributes>
+ <depends-on>
+ <property>
+ <type>capacity-scheduler</type>
+ <name>yarn.scheduler.capacity.root.queues</name>
+ </property>
+ </depends-on>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/ambari/blob/a03011c7/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
index 17ae78b..2b68442 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
@@ -348,17 +348,27 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
putHiveSiteProperty("hive.server2.enable.doAs", "true")
yarn_queues = "default"
- if "capacity-scheduler" in configurations and \
- "yarn.scheduler.capacity.root.queues" in configurations["capacity-scheduler"]["properties"]:
- yarn_queues = str(configurations["capacity-scheduler"]["properties"]["yarn.scheduler.capacity.root.queues"])
- putHiveSiteProperty("hive.server2.tez.default.queues", yarn_queues)
-
+ capacitySchedulerProperties = services['configurations']["capacity-scheduler"]["properties"] if "capacity-scheduler" in services['configurations'] else {}
+ if "yarn.scheduler.capacity.root.queues" in capacitySchedulerProperties:
+ yarn_queues = str(capacitySchedulerProperties["yarn.scheduler.capacity.root.queues"])
# Interactive Queues property attributes
putHiveServerPropertyAttribute = self.putPropertyAttribute(configurations, "hiveserver2-site")
- entries = []
- for queue in yarn_queues.split(","):
- entries.append({"label": str(queue) + " queue", "value": queue})
- putHiveSitePropertyAttribute("hive.server2.tez.default.queues", "entries", entries)
+ toProcessQueues = yarn_queues.split(",")
+ leafQueues = []
+ while len(toProcessQueues) > 0:
+ queue = toProcessQueues.pop();
+ queueKey = "yarn.scheduler.capacity.root." + queue + ".queues"
+ if queueKey in capacitySchedulerProperties:
+ # This is a parent queue - need to add children
+ subQueues = capacitySchedulerProperties[queueKey].split(",")
+ for subQueue in subQueues:
+ toProcessQueues.append(queue + "." + subQueue)
+ else:
+ # This is a leaf queue
+ leafQueues.append({"label": str(queue) + " queue", "value": queue})
+ leafQueues = sorted(leafQueues, key=lambda q:q['value'])
+ putHiveSitePropertyAttribute("hive.server2.tez.default.queues", "entries", leafQueues)
+ putHiveSiteProperty("hive.server2.tez.default.queues", ",".join([leafQueue['value'] for leafQueue in leafQueues]))
# Security
putHiveEnvProperty("hive_security_authorization", "None")
http://git-wip-us.apache.org/repos/asf/ambari/blob/a03011c7/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
index e242fce..a984a7a 100644
--- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
@@ -1181,6 +1181,65 @@ class TestHDP22StackAdvisor(TestCase):
self.stackAdvisor.recommendHIVEConfigurations(configurations, clusterData, services, hosts)
self.assertEquals(configurations, expected)
+ # test 'hive.server2.tez.default.queues' leaf queues
+ configurations['capacity-scheduler']['properties'] = {
+ "yarn.scheduler.capacity.maximum-am-resource-percent": "0.2",
+ "yarn.scheduler.capacity.maximum-applications": "10000",
+ "yarn.scheduler.capacity.node-locality-delay": "40",
+ "yarn.scheduler.capacity.queue-mappings-override.enable": "false",
+ "yarn.scheduler.capacity.resource-calculator": "org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator",
+ "yarn.scheduler.capacity.root.accessible-node-labels": "*",
+ "yarn.scheduler.capacity.root.acl_administer_queue": "*",
+ "yarn.scheduler.capacity.root.capacity": "100",
+ "yarn.scheduler.capacity.root.default.a.a1.acl_administer_queue": "*",
+ "yarn.scheduler.capacity.root.default.a.a1.acl_submit_applications": "*",
+ "yarn.scheduler.capacity.root.default.a.a1.capacity": "75",
+ "yarn.scheduler.capacity.root.default.a.a1.maximum-capacity": "100",
+ "yarn.scheduler.capacity.root.default.a.a1.minimum-user-limit-percent": "100",
+ "yarn.scheduler.capacity.root.default.a.a1.ordering-policy": "fifo",
+ "yarn.scheduler.capacity.root.default.a.a1.state": "RUNNING",
+ "yarn.scheduler.capacity.root.default.a.a1.user-limit-factor": "1",
+ "yarn.scheduler.capacity.root.default.a.a2.acl_administer_queue": "*",
+ "yarn.scheduler.capacity.root.default.a.a2.acl_submit_applications": "*",
+ "yarn.scheduler.capacity.root.default.a.a2.capacity": "25",
+ "yarn.scheduler.capacity.root.default.a.a2.maximum-capacity": "25",
+ "yarn.scheduler.capacity.root.default.a.a2.minimum-user-limit-percent": "100",
+ "yarn.scheduler.capacity.root.default.a.a2.ordering-policy": "fifo",
+ "yarn.scheduler.capacity.root.default.a.a2.state": "RUNNING",
+ "yarn.scheduler.capacity.root.default.a.a2.user-limit-factor": "1",
+ "yarn.scheduler.capacity.root.default.a.acl_administer_queue": "*",
+ "yarn.scheduler.capacity.root.default.a.acl_submit_applications": "*",
+ "yarn.scheduler.capacity.root.default.a.capacity": "50",
+ "yarn.scheduler.capacity.root.default.a.maximum-capacity": "100",
+ "yarn.scheduler.capacity.root.default.a.minimum-user-limit-percent": "100",
+ "yarn.scheduler.capacity.root.default.a.ordering-policy": "fifo",
+ "yarn.scheduler.capacity.root.default.a.queues": "a1,a2",
+ "yarn.scheduler.capacity.root.default.a.state": "RUNNING",
+ "yarn.scheduler.capacity.root.default.a.user-limit-factor": "1",
+ "yarn.scheduler.capacity.root.default.acl_submit_applications": "*",
+ "yarn.scheduler.capacity.root.default.b.acl_administer_queue": "*",
+ "yarn.scheduler.capacity.root.default.b.acl_submit_applications": "*",
+ "yarn.scheduler.capacity.root.default.b.capacity": "50",
+ "yarn.scheduler.capacity.root.default.b.maximum-capacity": "50",
+ "yarn.scheduler.capacity.root.default.b.minimum-user-limit-percent": "100",
+ "yarn.scheduler.capacity.root.default.b.ordering-policy": "fifo",
+ "yarn.scheduler.capacity.root.default.b.state": "RUNNING",
+ "yarn.scheduler.capacity.root.default.b.user-limit-factor": "1",
+ "yarn.scheduler.capacity.root.default.capacity": "100",
+ "yarn.scheduler.capacity.root.default.maximum-capacity": "100",
+ "yarn.scheduler.capacity.root.default.queues": "a,b",
+ "yarn.scheduler.capacity.root.default.state": "RUNNING",
+ "yarn.scheduler.capacity.root.default.user-limit-factor": "1",
+ "yarn.scheduler.capacity.root.queues": "default"
+ }
+ expected['hive-site']['properties']['hive.server2.tez.default.queues'] = 'default.a.a1,default.a.a2,default.b'
+ expected['hive-site']['property_attributes']['hive.server2.tez.default.queues'] = {
+ 'entries': [{'value': 'default.a.a1', 'label': 'default.a.a1 queue'}, {'value': 'default.a.a2', 'label': 'default.a.a2 queue'}, {'value': 'default.b', 'label': 'default.b queue'}]
+ }
+ self.stackAdvisor.recommendHIVEConfigurations(configurations, clusterData, services, hosts)
+ self.assertEquals(configurations['hive-site']['property_attributes']['hive.server2.tez.default.queues'], expected['hive-site']['property_attributes']['hive.server2.tez.default.queues'])
+ self.assertEquals(configurations['hive-site']['properties']['hive.server2.tez.default.queues'], expected['hive-site']['properties']['hive.server2.tez.default.queues'])
+
def test_recommendMapredConfigurationAttributesWithPigService(self):
configurations = {
"mapred-site": {