You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ao...@apache.org on 2016/02/23 11:42:21 UTC
[2/2] ambari git commit: AMBARI-15048. ambari should make sure it
uses only leaf queue name in configs (aonishuk)
AMBARI-15048. ambari should make sure it uses only leaf queue name in configs (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/167c147c
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/167c147c
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/167c147c
Branch: refs/heads/branch-2.2
Commit: 167c147c653efe0ca76b0ae488ae211ac2ef64be
Parents: 720b790
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Tue Feb 23 12:42:13 2016 +0200
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Tue Feb 23 12:42:13 2016 +0200
----------------------------------------------------------------------
.../stacks/HDP/2.2/services/stack_advisor.py | 6 ++--
.../services/HIVE/configuration/hive-site.xml | 6 ++++
.../stacks/HDPWIN/2.2/services/stack_advisor.py | 35 +++++++++++++++-----
.../stacks/2.2/common/test_stack_advisor.py | 4 +--
4 files changed, 38 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/167c147c/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
index 66569fd..f88eb0c 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
@@ -374,7 +374,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
# Interactive Queues property attributes
putHiveServerPropertyAttribute = self.putPropertyAttribute(configurations, "hiveserver2-site")
toProcessQueues = yarn_queues.split(",")
- leafQueues = []
+ leafQueueNames = set() # Remove duplicates
while len(toProcessQueues) > 0:
queue = toProcessQueues.pop()
queueKey = "yarn.scheduler.capacity.root." + queue + ".queues"
@@ -385,7 +385,9 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
toProcessQueues.append(queue + "." + subQueue)
else:
# This is a leaf queue
- leafQueues.append({"label": str(queue) + " queue", "value": queue})
+ queueName = queue.split(".")[-1] # Fully qualified queue name does not work, we should use only leaf name
+ leafQueueNames.add(queueName)
+ leafQueues = [{"label": str(queueName) + " queue", "value": queueName} for queueName in leafQueueNames]
leafQueues = sorted(leafQueues, key=lambda q:q['value'])
putHiveSitePropertyAttribute("hive.server2.tez.default.queues", "entries", leafQueues)
putHiveSiteProperty("hive.server2.tez.default.queues", ",".join([leafQueue['value'] for leafQueue in leafQueues]))
http://git-wip-us.apache.org/repos/asf/ambari/blob/167c147c/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/HIVE/configuration/hive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/HIVE/configuration/hive-site.xml b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/HIVE/configuration/hive-site.xml
index 8acc4dc..3059db6 100644
--- a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/HIVE/configuration/hive-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/HIVE/configuration/hive-site.xml
@@ -1292,6 +1292,12 @@ limitations under the License.
</entries>
<selection-cardinality>1+</selection-cardinality>
</value-attributes>
+ <depends-on>
+ <property>
+ <type>capacity-scheduler</type>
+ <name>yarn.scheduler.capacity.root.queues</name>
+ </property>
+ </depends-on>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/ambari/blob/167c147c/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py
index bf50aac..bd6276f 100644
--- a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py
@@ -373,17 +373,34 @@ class HDPWIN22StackAdvisor(HDPWIN21StackAdvisor):
putHiveSiteProperty("hive.server2.enable.doAs", "true")
yarn_queues = "default"
- if "capacity-scheduler" in configurations and \
- "yarn.scheduler.capacity.root.queues" in configurations["capacity-scheduler"]["properties"]:
- yarn_queues = str(configurations["capacity-scheduler"]["properties"]["yarn.scheduler.capacity.root.queues"])
- putHiveSiteProperty("hive.server2.tez.default.queues", yarn_queues)
-
+ capacitySchedulerProperties = {}
+ if "capacity-scheduler" in services['configurations'] and "capacity-scheduler" in services['configurations']["capacity-scheduler"]["properties"]:
+ properties = str(services['configurations']["capacity-scheduler"]["properties"]["capacity-scheduler"]).split('\n')
+ for property in properties:
+ key,sep,value = property.partition("=")
+ capacitySchedulerProperties[key] = value
+ if "yarn.scheduler.capacity.root.queues" in capacitySchedulerProperties:
+ yarn_queues = str(capacitySchedulerProperties["yarn.scheduler.capacity.root.queues"])
# Interactive Queues property attributes
putHiveServerPropertyAttribute = self.putPropertyAttribute(configurations, "hiveserver2-site")
- entries = []
- for queue in yarn_queues.split(","):
- entries.append({"label": str(queue) + " queue", "value": queue})
- putHiveSitePropertyAttribute("hive.server2.tez.default.queues", "entries", entries)
+ toProcessQueues = yarn_queues.split(",")
+ leafQueueNames = set() # Remove duplicates
+ while len(toProcessQueues) > 0:
+ queue = toProcessQueues.pop()
+ queueKey = "yarn.scheduler.capacity.root." + queue + ".queues"
+ if queueKey in capacitySchedulerProperties:
+ # This is a parent queue - need to add children
+ subQueues = capacitySchedulerProperties[queueKey].split(",")
+ for subQueue in subQueues:
+ toProcessQueues.append(queue + "." + subQueue)
+ else:
+ # This is a leaf queue
+ queueName = queue.split(".")[-1] # Fully qualified queue name does not work, we should use only leaf name
+ leafQueueNames.add(queueName)
+ leafQueues = [{"label": str(queueName) + " queue", "value": queueName} for queueName in leafQueueNames]
+ leafQueues = sorted(leafQueues, key=lambda q:q['value'])
+ putHiveSitePropertyAttribute("hive.server2.tez.default.queues", "entries", leafQueues)
+ putHiveSiteProperty("hive.server2.tez.default.queues", ",".join([leafQueue['value'] for leafQueue in leafQueues]))
# Security
putHiveEnvProperty("hive_security_authorization", "None")
http://git-wip-us.apache.org/repos/asf/ambari/blob/167c147c/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
index ade8db4..8d8938f 100644
--- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
@@ -1366,9 +1366,9 @@ class TestHDP22StackAdvisor(TestCase):
"yarn.scheduler.capacity.root.default.user-limit-factor=1\n"
"yarn.scheduler.capacity.root.queues=default"}
- expected['hive-site']['properties']['hive.server2.tez.default.queues'] = 'default.a.a1,default.a.a2,default.b'
+ expected['hive-site']['properties']['hive.server2.tez.default.queues'] = 'a1,a2,b'
expected['hive-site']['property_attributes']['hive.server2.tez.default.queues'] = {
- 'entries': [{'value': 'default.a.a1', 'label': 'default.a.a1 queue'}, {'value': 'default.a.a2', 'label': 'default.a.a2 queue'}, {'value': 'default.b', 'label': 'default.b queue'}]
+ 'entries': [{'value': 'a1', 'label': 'a1 queue'}, {'value': 'a2', 'label': 'a2 queue'}, {'value': 'b', 'label': 'b queue'}]
}
self.stackAdvisor.recommendHIVEConfigurations(configurations, clusterData, services, hosts)
self.assertEquals(configurations['hive-site']['property_attributes']['hive.server2.tez.default.queues'], expected['hive-site']['property_attributes']['hive.server2.tez.default.queues'])