You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by dm...@apache.org on 2016/06/30 16:58:50 UTC
ambari git commit: AMBARI-17503. Service check fails on non default
queue due to change in a way of leaf queue retrivieng (dgrinenko via
dlysnichenko)
Repository: ambari
Updated Branches:
refs/heads/branch-2.4 42a892ac3 -> 2fa199878
AMBARI-17503. Service check fails on non default queue due to change in a way of leaf queue retrivieng (dgrinenko via dlysnichenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2fa19987
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2fa19987
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2fa19987
Branch: refs/heads/branch-2.4
Commit: 2fa199878156b2da9834c62f8b4402a5debf9930
Parents: 42a892a
Author: Lisnichenko Dmitro <dl...@hortonworks.com>
Authored: Thu Jun 30 19:57:31 2016 +0300
Committer: Lisnichenko Dmitro <dl...@hortonworks.com>
Committed: Thu Jun 30 19:58:04 2016 +0300
----------------------------------------------------------------------
.../stacks/HDP/2.0.6/services/stack_advisor.py | 14 ++++-
.../stacks/2.5/common/test_stack_advisor.py | 54 +++++++++++++++++++-
2 files changed, 64 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/2fa19987/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index 9a310b5..06f7cfe 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -1360,7 +1360,12 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
if not propertyName in properties:
return self.getErrorItem("Value should be set")
capacity_scheduler_properties, received_as_key_value_pair = self.getCapacitySchedulerProperties(services)
- leafQueueNames = self.getAllYarnLeafQueues(capacity_scheduler_properties)
+ leafQueueNames = set()
+ leafQueues = self.getAllYarnLeafQueues(capacity_scheduler_properties)
+ for queue in leafQueues:
+ queue = queue.rstrip('.')
+ queueName = queue.split('.')[-1]
+ leafQueueNames.add(queueName)
value = properties[propertyName]
if len(leafQueueNames) == 0:
return None
@@ -1372,7 +1377,12 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
if services:
if 'configurations' in services:
capacity_scheduler_properties, received_as_key_value_pair = self.getCapacitySchedulerProperties(services)
- leafQueueNames = self.getAllYarnLeafQueues(capacity_scheduler_properties)
+ leafQueueNames = set()
+ leafQueues = self.getAllYarnLeafQueues(capacity_scheduler_properties)
+ for queue in leafQueues:
+ queue = queue.rstrip('.')
+ queueName = queue.split('.')[-1]
+ leafQueueNames.add(queueName)
if leafQueueNames:
return leafQueueNames.pop()
return "default"
http://git-wip-us.apache.org/repos/asf/ambari/blob/2fa19987/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
index 558d32d..a5bfefc 100644
--- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
@@ -7060,17 +7060,67 @@ class TestHDP25StackAdvisor(TestCase):
]
services = self.prepareServices(servicesInfo)
services["configurations"] = {"yarn-env":{"properties":{"service_check.queue.name": "default"}},
- "capacity-scheduler":{"properties":{"capacity-scheduler": "yarn.scheduler.capacity.root.queues=ndfqueue\n"}}}
+ "capacity-scheduler":{"properties":{
+ "capacity-scheduler": "yarn.scheduler.capacity.root.queues=ndfqueue,leaf\n" +
+ "yarn.scheduler.capacity.root.ndfqueue.queues=ndfqueue1,ndfqueue2\n"}}}
hosts = self.prepareHosts([])
result = self.stackAdvisor.validateConfigurations(services, hosts)
expectedItems = [
{'message': 'Queue is not exist, or not corresponds to existing YARN leaf queue', 'level': 'ERROR'}
]
self.assertValidationResult(expectedItems, result)
- services["configurations"]["yarn-env"]["properties"]["service_check.queue.name"] = "ndfqueue"
+ services["configurations"]["yarn-env"]["properties"]["service_check.queue.name"] = "ndfqueue2"
expectedItems = []
result = self.stackAdvisor.validateConfigurations(services, hosts)
self.assertValidationResult(expectedItems, result)
+ services["configurations"]["yarn-env"]["properties"]["service_check.queue.name"] = "leaf"
+ expectedItems = []
+ result = self.stackAdvisor.validateConfigurations(services, hosts)
+ self.assertValidationResult(expectedItems, result)
+
+ def test_recommendYARNQueueConfigurations(self):
+ configurations = {"yarn-env":{"properties":{"service_check.queue.name": "default"}},
+ "capacity-scheduler":{"properties":{
+ "capacity-scheduler": "yarn.scheduler.capacity.root.queues=ndfqueue\n" +
+ "yarn.scheduler.capacity.root.ndfqueue.queues=ndfqueue1,ndfqueue2\n"}}}
+ services = {"configurations": configurations, "services": [], "ambari-server-properties": {}}
+ clusterData = {
+ "containers" : 5,
+ "ramPerContainer": 256,
+ "mapMemory": 567,
+ "reduceMemory": 345.6666666666666,
+ "amMemory": 123.54,
+ "cpu": 4,
+ "referenceNodeManagerHost" : {
+ "total_mem" : 328960 * 1024
+ }
+ }
+ hosts = {
+ "items" : [
+ {
+ "href" : "/api/v1/hosts/c6401.ambari.apache.org",
+ "Hosts" : {
+ "cpu_count" : 1,
+ "host_name" : "c6401.ambari.apache.org",
+ "os_arch" : "x86_64",
+ "os_type" : "centos6",
+ "ph_cpu_count" : 1,
+ "public_host_name" : "c6401.ambari.apache.org",
+ "rack_info" : "/default-rack",
+ "total_mem" : 1922680
+ }
+ }
+ ]
+ }
+
+ self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, hosts)
+ self.stackAdvisor.recommendMapReduce2Configurations(configurations, clusterData, services, hosts)
+ self.stackAdvisor.recommendHIVEConfigurations(configurations, clusterData, services, hosts)
+ self.stackAdvisor.recommendTezConfigurations(configurations, clusterData, services, hosts)
+ self.assertEquals(configurations["yarn-env"]["properties"]["service_check.queue.name"], "ndfqueue2")
+ self.assertEquals(configurations["mapred-site"]["properties"]["mapreduce.job.queuename"], "ndfqueue2")
+ self.assertEquals(configurations["webhcat-site"]["properties"]["templeton.hadoop.queue.name"], "ndfqueue2")
+ self.assertEquals(configurations["tez-site"]["properties"]["tez.queue.name"], "ndfqueue2")
def assertValidationResult(self, expectedItems, result):
actualItems = []