You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by dm...@apache.org on 2016/08/05 16:29:50 UTC
[1/2] ambari git commit: AMBARI-18034. Issues with custom default
queue support (dgrinenko via dlysnichenko)
Repository: ambari
Updated Branches:
refs/heads/branch-2.4 d08da2d25 -> 7d2df9fff
refs/heads/trunk dc0722265 -> 2b7c97fc1
AMBARI-18034. Issues with custom default queue support (dgrinenko via dlysnichenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/7d2df9ff
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/7d2df9ff
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/7d2df9ff
Branch: refs/heads/branch-2.4
Commit: 7d2df9fff48a90b675672fbd1de817431f67e56a
Parents: d08da2d
Author: Lisnichenko Dmitro <dl...@hortonworks.com>
Authored: Fri Aug 5 19:27:45 2016 +0300
Committer: Lisnichenko Dmitro <dl...@hortonworks.com>
Committed: Fri Aug 5 19:27:45 2016 +0300
----------------------------------------------------------------------
.../server/upgrade/UpgradeCatalog240.java | 48 ------
.../0.12.0.2.0/configuration/webhcat-site.xml | 2 +-
.../4.0.0.2.0/package/scripts/params_linux.py | 2 +-
.../1.2.1/configuration/spark-defaults.xml | 2 +-
.../configuration/spark-thrift-sparkconf.xml | 2 +-
.../2.0.0/configuration/spark2-defaults.xml | 2 +-
.../configuration/spark2-thrift-sparkconf.xml | 2 +-
.../TEZ/0.4.0.2.1/configuration/tez-site.xml | 2 +-
.../configuration-mapred/mapred-site.xml | 2 +-
.../YARN/2.1.0.2.0/configuration/yarn-env.xml | 2 +-
.../2.1.0.2.0/package/scripts/params_linux.py | 2 +-
.../stacks/HDP/2.0.6/services/stack_advisor.py | 62 ++++----
.../stacks/HDP/2.1/services/stack_advisor.py | 4 +-
.../stacks/HDP/2.2/services/stack_advisor.py | 24 ++-
.../stacks/HDP/2.5/services/stack_advisor.py | 9 +-
.../server/upgrade/UpgradeCatalog240Test.java | 153 -------------------
.../stacks/2.0.6/common/test_stack_advisor.py | 2 +-
.../stacks/2.2/common/test_stack_advisor.py | 2 +-
.../stacks/2.3/common/test_stack_advisor.py | 6 +-
.../stacks/2.5/common/test_stack_advisor.py | 2 +-
20 files changed, 80 insertions(+), 252 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/7d2df9ff/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java
index 5c96e5a..8c22eeb 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java
@@ -192,10 +192,6 @@ public class UpgradeCatalog240 extends AbstractUpgradeCatalog {
protected static final String EXTENSION_LINK_ID_COLUMN = "link_id";
private static final Map<String, Integer> ROLE_ORDER;
- public static final String WEBHCAT_SITE_QUEUE_NAME = "templeton.hadoop.queue.name";
- public static final String TEZ_SITE_QUEUE_NAME = "tez.queue.name";
- public static final String YARN_ENV_QUEUE_NAME = "service_check.queue.name";
- public static final String MAPRED_SITE_QUEUE_NAME = "mapreduce.job.queuename";
private static final String AMS_HBASE_SITE = "ams-hbase-site";
private static final String HBASE_RPC_TIMEOUT_PROPERTY = "hbase.rpc.timeout";
private static final String AMS_HBASE_SITE_NORMALIZER_ENABLED_PROPERTY = "hbase.normalizer.enabled";
@@ -402,7 +398,6 @@ public class UpgradeCatalog240 extends AbstractUpgradeCatalog {
updateSparkConfigs();
updateHBaseConfigs();
updateFalconConfigs();
- updateQueueNameConfigs();
updateKerberosDescriptorArtifacts();
removeHiveOozieDBConnectionConfigs();
updateClustersAndHostsVersionStateTableDML();
@@ -2245,49 +2240,6 @@ public class UpgradeCatalog240 extends AbstractUpgradeCatalog {
}
/**
- * Updates the YARN Capacity Scheduler related configurations for the clusters managed by this Ambari
- * Update services configuration to set proper leaf queue name for services
- * update mapred-site, webhcat-site, tez-site, yarn-env
- *
- * @throws AmbariException if an error occurs while updating the configurations
- */
- protected void updateQueueNameConfigs() throws AmbariException {
- AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
- Clusters clusters = ambariManagementController.getClusters();
- Map<String, Cluster> clusterMap = getCheckedClusterMap(clusters);
-
- for (final Cluster cluster : clusterMap.values()) {
- Config capacityScheduler = cluster.getDesiredConfigByType(CAPACITY_SCHEDULER_CONFIG);
- if (capacityScheduler != null) {
- Map<String, String> capacitySchedulerProperties = capacityScheduler.getProperties();
- Set<String> leafQueues;
- leafQueues = getCapacitySchedulerLeafQueues(capacitySchedulerProperties);
- Set<String> installedServices = cluster.getServices().keySet();
- if (leafQueues ==null || leafQueues.isEmpty()) {
- LOG.warn("There is no leafQueues in capacity-scheduler");
- return;
- }
- if (installedServices.contains(cluster.getServiceByConfigType(WEBHCAT_SITE_CONFIG)) &&
- !isQueueNameValid(cluster, leafQueues, WEBHCAT_SITE_QUEUE_NAME, WEBHCAT_SITE_CONFIG)){
- updateQueueName(cluster, leafQueues, WEBHCAT_SITE_QUEUE_NAME, WEBHCAT_SITE_CONFIG);
- }
- if (installedServices.contains(cluster.getServiceByConfigType(TEZ_SITE_CONFIG)) &&
- !isQueueNameValid(cluster, leafQueues, TEZ_SITE_QUEUE_NAME, TEZ_SITE_CONFIG)){
- updateQueueName(cluster, leafQueues, TEZ_SITE_QUEUE_NAME, TEZ_SITE_CONFIG);
- }
- if (installedServices.contains(cluster.getServiceByConfigType(YARN_ENV_CONFIG)) &&
- !isQueueNameValid(cluster, leafQueues, YARN_ENV_QUEUE_NAME, YARN_ENV_CONFIG)){
- updateQueueName(cluster, leafQueues, YARN_ENV_QUEUE_NAME, YARN_ENV_CONFIG);
- }
- if (installedServices.contains(cluster.getServiceByConfigType(MAPRED_SITE_CONFIG)) &&
- !isQueueNameValid(cluster, leafQueues, MAPRED_SITE_QUEUE_NAME, MAPRED_SITE_CONFIG)){
- updateQueueName(cluster, leafQueues, MAPRED_SITE_QUEUE_NAME, MAPRED_SITE_CONFIG);
- }
- }
- }
- }
-
- /**
* Updates the Spark-related configurations for the clusters managed by this Ambari
* Removes falcon_store_uri from falcon-env.
* Updates {{hdp_full_version}} to {{full_stack_version}}
http://git-wip-us.apache.org/repos/asf/ambari/blob/7d2df9ff/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/webhcat-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/webhcat-site.xml b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/webhcat-site.xml
index d8012dd..cc5de73 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/webhcat-site.xml
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/webhcat-site.xml
@@ -155,6 +155,6 @@ limitations under the License.
<name>yarn.scheduler.capacity.root.queues</name>
</property>
</depends-on>
- <on-ambari-upgrade add="true"/>
+ <on-ambari-upgrade add="false"/>
</property>
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/7d2df9ff/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py
index ca57c81..db048bb 100644
--- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py
@@ -113,7 +113,7 @@ oozie_user = config['configurations']['oozie-env']['oozie_user']
smokeuser = config['configurations']['cluster-env']['smokeuser']
smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name']
smoke_hdfs_user_mode = 0770
-service_check_queue_name = default('/configurations/yarn-env/service_check.queue.name','default')
+service_check_queue_name = default('/configurations/yarn-env/service_check.queue.name', 'default')
# This config actually contains {oozie_user}
oozie_admin_users = format(config['configurations']['oozie-env']['oozie_admin_users'])
http://git-wip-us.apache.org/repos/asf/ambari/blob/7d2df9ff/ambari-server/src/main/resources/common-services/SPARK/1.2.1/configuration/spark-defaults.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/configuration/spark-defaults.xml b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/configuration/spark-defaults.xml
index ecb103c..748c270 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/configuration/spark-defaults.xml
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/configuration/spark-defaults.xml
@@ -75,7 +75,7 @@
<name>yarn.scheduler.capacity.root.queues</name>
</property>
</depends-on>
- <on-ambari-upgrade add="true"/>
+ <on-ambari-upgrade add="false"/>
</property>
<property>
<name>spark.yarn.containerLauncherMaxThreads</name>
http://git-wip-us.apache.org/repos/asf/ambari/blob/7d2df9ff/ambari-server/src/main/resources/common-services/SPARK/1.5.2/configuration/spark-thrift-sparkconf.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.5.2/configuration/spark-thrift-sparkconf.xml b/ambari-server/src/main/resources/common-services/SPARK/1.5.2/configuration/spark-thrift-sparkconf.xml
index 865aa73..42ed316 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.5.2/configuration/spark-thrift-sparkconf.xml
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.5.2/configuration/spark-thrift-sparkconf.xml
@@ -91,7 +91,7 @@
<name>yarn.scheduler.capacity.root.queues</name>
</property>
</depends-on>
- <on-ambari-upgrade add="true"/>
+ <on-ambari-upgrade add="false"/>
</property>
<property>
<name>spark.yarn.containerLauncherMaxThreads</name>
http://git-wip-us.apache.org/repos/asf/ambari/blob/7d2df9ff/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/configuration/spark2-defaults.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/configuration/spark2-defaults.xml b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/configuration/spark2-defaults.xml
index 2f9334e..4c28d46 100755
--- a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/configuration/spark2-defaults.xml
+++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/configuration/spark2-defaults.xml
@@ -31,7 +31,7 @@
<name>yarn.scheduler.capacity.root.queues</name>
</property>
</depends-on>
- <on-ambari-upgrade add="true"/>
+ <on-ambari-upgrade add="false"/>
</property>
<property>
<name>spark.driver.extraLibraryPath</name>
http://git-wip-us.apache.org/repos/asf/ambari/blob/7d2df9ff/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/configuration/spark2-thrift-sparkconf.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/configuration/spark2-thrift-sparkconf.xml b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/configuration/spark2-thrift-sparkconf.xml
index dfe87cb..6d73946 100755
--- a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/configuration/spark2-thrift-sparkconf.xml
+++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/configuration/spark2-thrift-sparkconf.xml
@@ -31,7 +31,7 @@
<name>yarn.scheduler.capacity.root.queues</name>
</property>
</depends-on>
- <on-ambari-upgrade add="true"/>
+ <on-ambari-upgrade add="false"/>
</property>
<property>
<name>spark.driver.extraLibraryPath</name>
http://git-wip-us.apache.org/repos/asf/ambari/blob/7d2df9ff/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/configuration/tez-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/configuration/tez-site.xml b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/configuration/tez-site.xml
index e7a851c..a39583c 100644
--- a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/configuration/tez-site.xml
+++ b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/configuration/tez-site.xml
@@ -243,6 +243,6 @@
<name>yarn.scheduler.capacity.root.queues</name>
</property>
</depends-on>
- <on-ambari-upgrade add="true"/>
+ <on-ambari-upgrade add="false"/>
</property>
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/7d2df9ff/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml
index 6951db0..434eea0 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml
@@ -476,6 +476,6 @@
<name>yarn.scheduler.capacity.root.queues</name>
</property>
</depends-on>
- <on-ambari-upgrade add="true"/>
+ <on-ambari-upgrade add="false"/>
</property>
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/7d2df9ff/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-env.xml b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-env.xml
index 152c463..c3bbcb6 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-env.xml
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-env.xml
@@ -255,6 +255,6 @@ YARN_OPTS="$YARN_OPTS -Djava.io.tmpdir={{hadoop_java_io_tmpdir}}"
<name>yarn.scheduler.capacity.root.queues</name>
</property>
</depends-on>
- <on-ambari-upgrade add="true"/>
+ <on-ambari-upgrade add="false"/>
</property>
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/7d2df9ff/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
index b2c635d..4d42861 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
@@ -183,7 +183,7 @@ mapred_log_dir_prefix = config['configurations']['mapred-env']['mapred_log_dir_p
mapred_env_sh_template = config['configurations']['mapred-env']['content']
yarn_env_sh_template = config['configurations']['yarn-env']['content']
yarn_nodemanager_recovery_dir = default('/configurations/yarn-site/yarn.nodemanager.recovery.dir', None)
-service_check_queue_name = default('/configurations/yarn-env/service_check.queue.name','default')
+service_check_queue_name = default('/configurations/yarn-env/service_check.queue.name', 'default')
if len(rm_hosts) > 1:
additional_rm_host = rm_hosts[1]
http://git-wip-us.apache.org/repos/asf/ambari/blob/7d2df9ff/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index 2efb60d..87618a8 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -113,7 +113,11 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
putYarnProperty('yarn.scheduler.minimum-allocation-mb', int(clusterData['ramPerContainer']))
putYarnProperty('yarn.scheduler.maximum-allocation-mb', int(configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"]))
putYarnEnvProperty('min_user_id', self.get_system_min_uid())
- putYarnEnvProperty("service_check.queue.name", self.recommendYarnQueue(services))
+
+ sc_queue_name = self.recommendYarnQueue(services, "yarn-env", "service_check.queue.name")
+ if sc_queue_name is not None:
+ putYarnEnvProperty("service_check.queue.name", sc_queue_name)
+
containerExecutorGroup = 'hadoop'
if 'cluster-env' in services['configurations'] and 'user_group' in services['configurations']['cluster-env']['properties']:
containerExecutorGroup = services['configurations']['cluster-env']['properties']['user_group']
@@ -140,7 +144,9 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
putMapredProperty('mapreduce.map.java.opts', "-Xmx" + str(int(round(0.8 * clusterData['mapMemory']))) + "m")
putMapredProperty('mapreduce.reduce.java.opts', "-Xmx" + str(int(round(0.8 * clusterData['reduceMemory']))) + "m")
putMapredProperty('mapreduce.task.io.sort.mb', min(int(round(0.4 * clusterData['mapMemory'])), 1024))
- putMapredProperty("mapreduce.job.queuename", self.recommendYarnQueue(services))
+ mr_queue = self.recommendYarnQueue(services, "mapred-site", "mapreduce.job.queuename")
+ if mr_queue is not None:
+ putMapredProperty("mapreduce.job.queuename", mr_queue)
def getAmbariUser(self, services):
ambari_user = services['ambari-server-properties']['ambari-server.user']
@@ -1382,34 +1388,36 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
return None
def validatorYarnQueue(self, properties, recommendedDefaults, propertyName, services):
- if not propertyName in properties:
+ if propertyName not in properties:
return self.getErrorItem("Value should be set")
- capacity_scheduler_properties, received_as_key_value_pair = self.getCapacitySchedulerProperties(services)
- leafQueueNames = set()
- leafQueues = self.getAllYarnLeafQueues(capacity_scheduler_properties)
- for queue in leafQueues:
- queue = queue.rstrip('.')
- queueName = queue.split('.')[-1]
- leafQueueNames.add(queueName)
- value = properties[propertyName]
- if len(leafQueueNames) == 0:
+
+ capacity_scheduler_properties, _ = self.getCapacitySchedulerProperties(services)
+ leaf_queue_names = self.getAllYarnLeafQueues(capacity_scheduler_properties)
+ queue_name = properties[propertyName]
+
+ if len(leaf_queue_names) == 0:
return None
- if value not in leafQueueNames:
- return self.getErrorItem("Queue is not exist, or not corresponds to existing YARN leaf queue")
+ elif queue_name not in leaf_queue_names:
+ return self.getErrorItem("Queue is not exist or not corresponds to existing YARN leaf queue")
+
return None
- def recommendYarnQueue(self, services):
- if services:
- if 'configurations' in services:
- capacity_scheduler_properties, received_as_key_value_pair = self.getCapacitySchedulerProperties(services)
- leafQueueNames = set()
- leafQueues = self.getAllYarnLeafQueues(capacity_scheduler_properties)
- for queue in leafQueues:
- queue = queue.rstrip('.')
- queueName = queue.split('.')[-1]
- leafQueueNames.add(queueName)
- if leafQueueNames:
- return leafQueueNames.pop()
+ def recommendYarnQueue(self, services, catalog_name=None, queue_property=None):
+ old_queue_name = None
+
+ if services and 'configurations' in services:
+ configurations = services["configurations"]
+ if catalog_name in configurations and queue_property in configurations[catalog_name]["properties"]:
+ old_queue_name = configurations[catalog_name]["properties"][queue_property]
+
+ capacity_scheduler_properties, _ = self.getCapacitySchedulerProperties(services)
+ leaf_queues = sorted(self.getAllYarnLeafQueues(capacity_scheduler_properties))
+
+ if leaf_queues and (old_queue_name is None or old_queue_name not in leaf_queues):
+ return leaf_queues.pop()
+ elif old_queue_name and old_queue_name in leaf_queues:
+ return None
+
return "default"
def validateXmxValue(self, properties, recommendedDefaults, propertyName):
@@ -1641,7 +1649,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
"""
def getAllYarnLeafQueues(self, capacitySchedulerProperties):
config_list = capacitySchedulerProperties.keys()
- yarn_queues = []
+ yarn_queues = None
leafQueueNames = set()
if 'yarn.scheduler.capacity.root.queues' in config_list:
yarn_queues = capacitySchedulerProperties.get('yarn.scheduler.capacity.root.queues')
http://git-wip-us.apache.org/repos/asf/ambari/blob/7d2df9ff/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
index 6ec4b8d..34b4614 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
@@ -127,7 +127,9 @@ class HDP21StackAdvisor(HDP206StackAdvisor):
putTezProperty("tez.am.java.opts",
"-server -Xmx" + str(int(0.8 * clusterData["amMemory"]))
+ "m -Djava.net.preferIPv4Stack=true -XX:+UseNUMA -XX:+UseParallelGC")
- putTezProperty("tez.queue.name", self.recommendYarnQueue(services))
+ recommended_tez_queue = self.recommendYarnQueue(services, "tez-site", "tez.queue.name")
+ if recommended_tez_queue is not None:
+ putTezProperty("tez.queue.name", recommended_tez_queue)
def getNotPreferableOnServerComponents(self):
http://git-wip-us.apache.org/repos/asf/ambari/blob/7d2df9ff/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
index 9ec1f05..4854514 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
@@ -57,12 +57,17 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
:type hosts dict
"""
putSparkProperty = self.putProperty(configurations, "spark-defaults", services)
- putSparkProperty("spark.yarn.queue", self.recommendYarnQueue(services))
+
+ spark_queue = self.recommendYarnQueue(services, "spark-defaults", "spark.yarn.queue")
+ if spark_queue is not None:
+ putSparkProperty("spark.yarn.queue", spark_queue)
# add only if spark supports this config
if "configurations" in services and "spark-thrift-sparkconf" in services["configurations"]:
putSparkThriftSparkConf = self.putProperty(configurations, "spark-thrift-sparkconf", services)
- putSparkThriftSparkConf("spark.yarn.queue", self.recommendYarnQueue(services))
+ recommended_spark_queue = self.recommendYarnQueue(services, "spark-thrift-sparkconf", "spark.yarn.queue")
+ if recommended_spark_queue is not None:
+ putSparkThriftSparkConf("spark.yarn.queue", recommended_spark_queue)
def recommendYARNConfigurations(self, configurations, clusterData, services, hosts):
@@ -445,7 +450,10 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
leafQueues = sorted(leafQueues, key=lambda q:q['value'])
putHiveSitePropertyAttribute("hive.server2.tez.default.queues", "entries", leafQueues)
putHiveSiteProperty("hive.server2.tez.default.queues", ",".join([leafQueue['value'] for leafQueue in leafQueues]))
- putWebhcatSiteProperty("templeton.hadoop.queue.name", self.recommendYarnQueue(services))
+
+ webhcat_queue = self.recommendYarnQueue(services, "webhcat-site", "templeton.hadoop.queue.name")
+ if webhcat_queue is not None:
+ putWebhcatSiteProperty("templeton.hadoop.queue.name", webhcat_queue)
# Recommend Ranger Hive authorization as per Ranger Hive plugin property
@@ -812,7 +820,10 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
putTezProperty("tez.runtime.io.sort.mb", min(int(taskResourceMemory * 0.4), 2047))
putTezProperty("tez.runtime.unordered.output.buffer.size-mb", int(taskResourceMemory * 0.075))
putTezProperty("tez.session.am.dag.submit.timeout.secs", "600")
- putTezProperty("tez.queue.name", self.recommendYarnQueue(services))
+
+ tez_queue = self.recommendYarnQueue(services, "tez-site", "tez.queue.name")
+ if tez_queue is not None:
+ putTezProperty("tez.queue.name", tez_queue)
serverProperties = services["ambari-server-properties"]
latest_tez_jar_version = None
@@ -1079,7 +1090,10 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
putMapredPropertyAttribute("yarn.app.mapreduce.am.resource.mb", "minimum", yarnMinAllocationSize)
# Hadoop MR limitation
putMapredPropertyAttribute("mapreduce.task.io.sort.mb", "maximum", "2047")
- putMapredProperty("mapreduce.job.queuename", self.recommendYarnQueue(services))
+
+ mr_queue = self.recommendYarnQueue(services, "mapred-site", "mapreduce.job.queuename")
+ if mr_queue is not None:
+ putMapredProperty("mapreduce.job.queuename", mr_queue)
def validateMapReduce2Configurations(self, properties, recommendedDefaults, configurations, services, hosts):
validationItems = [ {"config-name": 'mapreduce.map.java.opts', "item": self.validateXmxValue(properties, recommendedDefaults, 'mapreduce.map.java.opts')},
http://git-wip-us.apache.org/repos/asf/ambari/blob/7d2df9ff/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
index 68b03ad..e982285 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
@@ -405,8 +405,13 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
putSparkProperty = self.putProperty(configurations, "spark2-defaults", services)
putSparkThriftSparkConf = self.putProperty(configurations, "spark2-thrift-sparkconf", services)
- putSparkProperty("spark.yarn.queue", self.recommendYarnQueue(services))
- putSparkThriftSparkConf("spark.yarn.queue", self.recommendYarnQueue(services))
+ spark_queue = self.recommendYarnQueue(services, "spark2-defaults", "spark.yarn.queue")
+ if spark_queue is not None:
+ putSparkProperty("spark.yarn.queue", spark_queue)
+
+ spart_thrift_queue = self.recommendYarnQueue(services, "spark2-thrift-sparkconf", "spark.yarn.queue")
+ if spart_thrift_queue is not None:
+ putSparkThriftSparkConf("spark.yarn.queue", spart_thrift_queue)
def recommendStormConfigurations(self, configurations, clusterData, services, hosts):
super(HDP25StackAdvisor, self).recommendStormConfigurations(configurations, clusterData, services, hosts)
http://git-wip-us.apache.org/repos/asf/ambari/blob/7d2df9ff/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java
index f7c33ad..3b42984 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java
@@ -564,7 +564,6 @@ public class UpgradeCatalog240Test {
Method updateHostRoleCommandTableDML = UpgradeCatalog240.class.getDeclaredMethod("updateHostRoleCommandTableDML");
Method updateKerberosEnv = UpgradeCatalog240.class.getDeclaredMethod("updateKerberosConfigs");
Method updateYarnEnv = UpgradeCatalog240.class.getDeclaredMethod("updateYarnEnv");
- Method updateQueueNameConfigs = UpgradeCatalog240.class.getDeclaredMethod("updateQueueNameConfigs");
Method removeHiveOozieDBConnectionConfigs = UpgradeCatalog240.class.getDeclaredMethod("removeHiveOozieDBConnectionConfigs");
Method updateClustersAndHostsVersionStateTableDML = UpgradeCatalog240.class.getDeclaredMethod("updateClustersAndHostsVersionStateTableDML");
Method removeStandardDeviationAlerts = UpgradeCatalog240.class.getDeclaredMethod("removeStandardDeviationAlerts");
@@ -614,7 +613,6 @@ public class UpgradeCatalog240Test {
.addMockedMethod(updateHostRoleCommandTableDML)
.addMockedMethod(updateKerberosEnv)
.addMockedMethod(updateYarnEnv)
- .addMockedMethod(updateQueueNameConfigs)
.addMockedMethod(removeHiveOozieDBConnectionConfigs)
.addMockedMethod(updateClustersAndHostsVersionStateTableDML)
.addMockedMethod(removeStandardDeviationAlerts)
@@ -655,7 +653,6 @@ public class UpgradeCatalog240Test {
upgradeCatalog240.updateHostRoleCommandTableDML();
upgradeCatalog240.updateKerberosConfigs();
upgradeCatalog240.updateYarnEnv();
- upgradeCatalog240.updateQueueNameConfigs();
upgradeCatalog240.removeHiveOozieDBConnectionConfigs();
upgradeCatalog240.updateClustersAndHostsVersionStateTableDML();
upgradeCatalog240.removeStandardDeviationAlerts();
@@ -1147,156 +1144,6 @@ public class UpgradeCatalog240Test {
assertTrue(Maps.difference(newPropertiesYarnEnv, updatedProperties).areEqual());
}
- /**
- * Test that queue names updated in mapred-site, webhcat-site, tez-site, yarn-env
- * @throws Exception
- */
- @SuppressWarnings("unchecked")
- @Test
- public void testQueueNameUpdateConfigs() throws Exception{
- Map<String, String> oldPropertiesCapacityScheduler = new HashMap<String, String>() {
- {
- put("yarn.scheduler.capacity.root.queues", "default3,d3");
- put("yarn.scheduler.capacity.root.d3.queues", "default4");
- put("yarn.scheduler.capacity.root.default3.queues", "default5");
- }
- };
- Map<String, String> oldPropertiesTezSite = new HashMap<String, String>() {
- {
- put("tez.queue.name", "default3");
- }
- };
- Map<String, String> oldPropertiesWebhcatSite = new HashMap<String, String>() {
- {
- put("templeton.hadoop.queue.name", "default3");
- }
- };
- Map<String, String> oldPropertiesMapredSite = new HashMap<String, String>() {
- {
- put("mapreduce.job.queuename", "default3");
- }
- };
- Map<String, String> oldPropertiesYarnEnv = new HashMap<String, String>() {
- {
- put("service_check.queue.name", "default3");
- }
- };
-
- Map<String, String> newPropertiesTezSite = new HashMap<String, String>() {
- {
- put("tez.queue.name", "default5");
- }
- };
- Map<String, String> newPropertiesWebhcatSite = new HashMap<String, String>() {
- {
- put("templeton.hadoop.queue.name", "default5");
- }
- };
- Map<String, String> newPropertiesMapredSite = new HashMap<String, String>() {
- {
- put("mapreduce.job.queuename", "default5");
- }
- };
- Map<String, String> newPropertiesYarnEnv = new HashMap<String, String>() {
- {
- put("service_check.queue.name", "default5");
- }
- };
-
- EasyMockSupport easyMockSupport = new EasyMockSupport();
-
- Clusters clusters = easyMockSupport.createNiceMock(Clusters.class);
- final Cluster cluster = easyMockSupport.createNiceMock(Cluster.class);
- Config mockCapacityScheduler = easyMockSupport.createNiceMock(Config.class);
- Config mockWebhcatSite = easyMockSupport.createNiceMock(Config.class);
- Config mockTezSite = easyMockSupport.createNiceMock(Config.class);
- Config mockMapredSite = easyMockSupport.createNiceMock(Config.class);
- Config mockYarnEnv = easyMockSupport.createNiceMock(Config.class);
-
- expect(clusters.getClusters()).andReturn(new HashMap<String, Cluster>() {{
- put("normal", cluster);
- }}).anyTimes();
-
- expect(cluster.getServices()).andReturn(new HashMap<String, Service>() {
- {
- put("YARN", null);
- put("HIVE", null);
- put("MAPREDUCE", null);
- put("TEZ", null);
- }
- }).atLeastOnce();
-
- expect(cluster.getServiceByConfigType(WEBHCAT_SITE_CONFIG_TYPE)).andReturn("HIVE").atLeastOnce();
- expect(cluster.getServiceByConfigType(TEZ_SITE_CONFIG_TYPE)).andReturn("TEZ").atLeastOnce();
- expect(cluster.getServiceByConfigType(MAPRED_SITE_CONFIG_TYPE)).andReturn("MAPREDUCE").atLeastOnce();
- expect(cluster.getServiceByConfigType(YARN_ENV_CONFIG_TYPE)).andReturn("YARN").atLeastOnce();
-
- expect(cluster.getDesiredConfigByType(CAPACITY_SCHEDULER_CONFIG_TYPE)).andReturn(mockCapacityScheduler).atLeastOnce();
- expect(cluster.getDesiredConfigByType(WEBHCAT_SITE_CONFIG_TYPE)).andReturn(mockWebhcatSite).atLeastOnce();
- expect(cluster.getDesiredConfigByType(TEZ_SITE_CONFIG_TYPE)).andReturn(mockTezSite).atLeastOnce();
- expect(cluster.getDesiredConfigByType(MAPRED_SITE_CONFIG_TYPE)).andReturn(mockMapredSite).atLeastOnce();
- expect(cluster.getDesiredConfigByType(YARN_ENV_CONFIG_TYPE)).andReturn(mockYarnEnv).atLeastOnce();
-
-
-
- expect(mockCapacityScheduler.getProperties()).andReturn(oldPropertiesCapacityScheduler).anyTimes();
- expect(mockWebhcatSite.getProperties()).andReturn(oldPropertiesWebhcatSite).anyTimes();
- expect(mockTezSite.getProperties()).andReturn(oldPropertiesTezSite).anyTimes();
- expect(mockMapredSite.getProperties()).andReturn(oldPropertiesMapredSite).anyTimes();
- expect(mockYarnEnv.getProperties()).andReturn(oldPropertiesYarnEnv).anyTimes();
-
- Injector injector = easyMockSupport.createNiceMock(Injector.class);
- expect(injector.getInstance(Gson.class)).andReturn(null).anyTimes();
- expect(injector.getInstance(MaintenanceStateHelper.class)).andReturn(null).anyTimes();
- expect(injector.getInstance(KerberosHelper.class)).andReturn(createNiceMock(KerberosHelper.class)).anyTimes();
-
- replay(injector, clusters, mockCapacityScheduler, mockWebhcatSite, mockTezSite, mockMapredSite,
- mockYarnEnv, cluster);
-
- AmbariManagementControllerImpl controller = createMockBuilder(AmbariManagementControllerImpl.class)
- .addMockedMethod("createConfiguration")
- .addMockedMethod("getClusters", new Class[] { })
- .addMockedMethod("createConfig")
- .withConstructor(createNiceMock(ActionManager.class), clusters, injector)
- .createNiceMock();
-
- Injector injector2 = easyMockSupport.createNiceMock(Injector.class);
- Capture<Map> propertiesCapacityScheduler = EasyMock.newCapture();
- Capture<Map> propertiesCaptureYarnEnv = EasyMock.newCapture();
- Capture<Map> propertiesCaptureWebhcatSite = EasyMock.newCapture();
- Capture<Map> propertiesCaptureTezSite= EasyMock.newCapture();
- Capture<Map> propertiesCaptureMapredSite = EasyMock.newCapture();
-
- expect(injector2.getInstance(AmbariManagementController.class)).andReturn(controller).anyTimes();
- expect(controller.getClusters()).andReturn(clusters).anyTimes();
- expect(controller.createConfig(anyObject(Cluster.class), eq(CAPACITY_SCHEDULER_CONFIG_TYPE), capture(propertiesCapacityScheduler), anyString(),
- anyObject(Map.class))).andReturn(createNiceMock(Config.class)).once();
- expect(controller.createConfig(anyObject(Cluster.class), eq(YARN_ENV_CONFIG_TYPE), capture(propertiesCaptureYarnEnv), anyString(),
- anyObject(Map.class))).andReturn(createNiceMock(Config.class)).once();
- expect(controller.createConfig(anyObject(Cluster.class), eq(WEBHCAT_SITE_CONFIG_TYPE), capture(propertiesCaptureWebhcatSite), anyString(),
- anyObject(Map.class))).andReturn(createNiceMock(Config.class)).once();
- expect(controller.createConfig(anyObject(Cluster.class), eq(TEZ_SITE_CONFIG_TYPE), capture(propertiesCaptureTezSite), anyString(),
- anyObject(Map.class))).andReturn(createNiceMock(Config.class)).once();
- expect(controller.createConfig(anyObject(Cluster.class), eq(MAPRED_SITE_CONFIG_TYPE), capture(propertiesCaptureMapredSite), anyString(),
- anyObject(Map.class))).andReturn(createNiceMock(Config.class)).once();
-
- replay(controller, injector2);
- new UpgradeCatalog240(injector2).updateQueueNameConfigs();
- easyMockSupport.verifyAll();
-
- Map<String, String> updatedPropertiesYarnEnv = propertiesCaptureYarnEnv.getValue();
- assertTrue(Maps.difference(newPropertiesYarnEnv, updatedPropertiesYarnEnv).areEqual());
-
- Map<String, String> updatedPropertiesWebhcatSite = propertiesCaptureWebhcatSite.getValue();
- assertTrue(Maps.difference(newPropertiesWebhcatSite, updatedPropertiesWebhcatSite).areEqual());
-
- Map<String, String> updatedPropertiesTezSite = propertiesCaptureTezSite.getValue();
- assertTrue(Maps.difference(newPropertiesTezSite, updatedPropertiesTezSite).areEqual());
-
- Map<String, String> updatedPropertiesMapredSite = propertiesCaptureMapredSite.getValue();
- assertTrue(Maps.difference(newPropertiesMapredSite, updatedPropertiesMapredSite).areEqual());
- }
-
@Test
public void testSparkConfigUpdate() throws Exception{
http://git-wip-us.apache.org/repos/asf/ambari/blob/7d2df9ff/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
index 4a12920..e28389e 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
@@ -222,7 +222,7 @@ class TestHDP206StackAdvisor(TestCase):
hosts = self.prepareHosts([])
result = self.stackAdvisor.validateConfigurations(services, hosts)
expectedItems = [
- {'message': 'Queue is not exist, or not corresponds to existing YARN leaf queue', 'level': 'ERROR'}
+ {'message': 'Queue is not exist or not corresponds to existing YARN leaf queue', 'level': 'ERROR'}
]
self.assertValidationResult(expectedItems, result)
services["configurations"]["yarn-env"]["properties"]["service_check.queue.name"] = "ndfqueue"
http://git-wip-us.apache.org/repos/asf/ambari/blob/7d2df9ff/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
index 75a26d2..c3c039b 100644
--- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
@@ -1283,7 +1283,7 @@ class TestHDP22StackAdvisor(TestCase):
},
'webhcat-site': {
'properties': {
- 'templeton.hadoop.queue.name': 'queue1'
+ 'templeton.hadoop.queue.name': 'queue2'
}
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/7d2df9ff/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
index 272269d..137fd47 100644
--- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
@@ -657,7 +657,7 @@ class TestHDP23StackAdvisor(TestCase):
},
'webhcat-site': {
'properties': {
- 'templeton.hadoop.queue.name': 'queue1'
+ 'templeton.hadoop.queue.name': 'queue2'
}
}
}
@@ -933,7 +933,7 @@ class TestHDP23StackAdvisor(TestCase):
},
'webhcat-site': {
'properties': {
- 'templeton.hadoop.queue.name': 'queue1'
+ 'templeton.hadoop.queue.name': 'queue2'
}
}
}
@@ -1163,7 +1163,7 @@ class TestHDP23StackAdvisor(TestCase):
"tez.session.am.dag.submit.timeout.secs": "600",
"tez.runtime.unordered.output.buffer.size-mb": "57",
"tez.am.resource.memory.mb": "4000",
- "tez.queue.name": "queue1",
+ "tez.queue.name": "queue2",
}
},
"yarn-site": {
http://git-wip-us.apache.org/repos/asf/ambari/blob/7d2df9ff/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
index d1f6d88..0ed1761 100644
--- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
@@ -7252,7 +7252,7 @@ class TestHDP25StackAdvisor(TestCase):
hosts = self.prepareHosts([])
result = self.stackAdvisor.validateConfigurations(services, hosts)
expectedItems = [
- {'message': 'Queue is not exist, or not corresponds to existing YARN leaf queue', 'level': 'ERROR'}
+ {'message': 'Queue is not exist or not corresponds to existing YARN leaf queue', 'level': 'ERROR'}
]
self.assertValidationResult(expectedItems, result)
services["configurations"]["yarn-env"]["properties"]["service_check.queue.name"] = "ndfqueue2"
[2/2] ambari git commit: AMBARI-18034. Issues with custom default
queue support (dgrinenko via dlysnichenko)
Posted by dm...@apache.org.
AMBARI-18034. Issues with custom default queue support (dgrinenko via dlysnichenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2b7c97fc
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2b7c97fc
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2b7c97fc
Branch: refs/heads/trunk
Commit: 2b7c97fc19a57bb28e8d850242d573fdf1915174
Parents: dc07222
Author: Lisnichenko Dmitro <dl...@hortonworks.com>
Authored: Fri Aug 5 19:27:45 2016 +0300
Committer: Lisnichenko Dmitro <dl...@hortonworks.com>
Committed: Fri Aug 5 19:29:24 2016 +0300
----------------------------------------------------------------------
.../server/upgrade/UpgradeCatalog240.java | 48 ------
.../0.12.0.2.0/configuration/webhcat-site.xml | 2 +-
.../4.0.0.2.0/package/scripts/params_linux.py | 2 +-
.../1.2.1/configuration/spark-defaults.xml | 2 +-
.../configuration/spark-thrift-sparkconf.xml | 2 +-
.../2.0.0/configuration/spark2-defaults.xml | 2 +-
.../configuration/spark2-thrift-sparkconf.xml | 2 +-
.../TEZ/0.4.0.2.1/configuration/tez-site.xml | 2 +-
.../configuration-mapred/mapred-site.xml | 2 +-
.../YARN/2.1.0.2.0/configuration/yarn-env.xml | 2 +-
.../2.1.0.2.0/package/scripts/params_linux.py | 2 +-
.../stacks/HDP/2.0.6/services/stack_advisor.py | 62 ++++----
.../stacks/HDP/2.1/services/stack_advisor.py | 4 +-
.../stacks/HDP/2.2/services/stack_advisor.py | 24 ++-
.../stacks/HDP/2.5/services/stack_advisor.py | 9 +-
.../server/upgrade/UpgradeCatalog240Test.java | 153 -------------------
.../stacks/2.0.6/common/test_stack_advisor.py | 2 +-
.../stacks/2.2/common/test_stack_advisor.py | 2 +-
.../stacks/2.3/common/test_stack_advisor.py | 6 +-
.../stacks/2.5/common/test_stack_advisor.py | 2 +-
20 files changed, 80 insertions(+), 252 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/2b7c97fc/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java
index 1382b92..47f51a7 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java
@@ -192,10 +192,6 @@ public class UpgradeCatalog240 extends AbstractUpgradeCatalog {
protected static final String EXTENSION_LINK_ID_COLUMN = "link_id";
private static final Map<String, Integer> ROLE_ORDER;
- public static final String WEBHCAT_SITE_QUEUE_NAME = "templeton.hadoop.queue.name";
- public static final String TEZ_SITE_QUEUE_NAME = "tez.queue.name";
- public static final String YARN_ENV_QUEUE_NAME = "service_check.queue.name";
- public static final String MAPRED_SITE_QUEUE_NAME = "mapreduce.job.queuename";
private static final String AMS_HBASE_SITE = "ams-hbase-site";
private static final String HBASE_RPC_TIMEOUT_PROPERTY = "hbase.rpc.timeout";
private static final String AMS_HBASE_SITE_NORMALIZER_ENABLED_PROPERTY = "hbase.normalizer.enabled";
@@ -402,7 +398,6 @@ public class UpgradeCatalog240 extends AbstractUpgradeCatalog {
updateSparkConfigs();
updateHBaseConfigs();
updateFalconConfigs();
- updateQueueNameConfigs();
updateKerberosDescriptorArtifacts();
removeHiveOozieDBConnectionConfigs();
updateClustersAndHostsVersionStateTableDML();
@@ -2260,49 +2255,6 @@ public class UpgradeCatalog240 extends AbstractUpgradeCatalog {
}
/**
- * Updates the YARN Capacity Scheduler related configurations for the clusters managed by this Ambari
- * Update services configuration to set proper leaf queue name for services
- * update mapred-site, webhcat-site, tez-site, yarn-env
- *
- * @throws AmbariException if an error occurs while updating the configurations
- */
- protected void updateQueueNameConfigs() throws AmbariException {
- AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
- Clusters clusters = ambariManagementController.getClusters();
- Map<String, Cluster> clusterMap = getCheckedClusterMap(clusters);
-
- for (final Cluster cluster : clusterMap.values()) {
- Config capacityScheduler = cluster.getDesiredConfigByType(CAPACITY_SCHEDULER_CONFIG);
- if (capacityScheduler != null) {
- Map<String, String> capacitySchedulerProperties = capacityScheduler.getProperties();
- Set<String> leafQueues;
- leafQueues = getCapacitySchedulerLeafQueues(capacitySchedulerProperties);
- Set<String> installedServices = cluster.getServices().keySet();
- if (leafQueues ==null || leafQueues.isEmpty()) {
- LOG.warn("There is no leafQueues in capacity-scheduler");
- return;
- }
- if (installedServices.contains(cluster.getServiceByConfigType(WEBHCAT_SITE_CONFIG)) &&
- !isQueueNameValid(cluster, leafQueues, WEBHCAT_SITE_QUEUE_NAME, WEBHCAT_SITE_CONFIG)){
- updateQueueName(cluster, leafQueues, WEBHCAT_SITE_QUEUE_NAME, WEBHCAT_SITE_CONFIG);
- }
- if (installedServices.contains(cluster.getServiceByConfigType(TEZ_SITE_CONFIG)) &&
- !isQueueNameValid(cluster, leafQueues, TEZ_SITE_QUEUE_NAME, TEZ_SITE_CONFIG)){
- updateQueueName(cluster, leafQueues, TEZ_SITE_QUEUE_NAME, TEZ_SITE_CONFIG);
- }
- if (installedServices.contains(cluster.getServiceByConfigType(YARN_ENV_CONFIG)) &&
- !isQueueNameValid(cluster, leafQueues, YARN_ENV_QUEUE_NAME, YARN_ENV_CONFIG)){
- updateQueueName(cluster, leafQueues, YARN_ENV_QUEUE_NAME, YARN_ENV_CONFIG);
- }
- if (installedServices.contains(cluster.getServiceByConfigType(MAPRED_SITE_CONFIG)) &&
- !isQueueNameValid(cluster, leafQueues, MAPRED_SITE_QUEUE_NAME, MAPRED_SITE_CONFIG)){
- updateQueueName(cluster, leafQueues, MAPRED_SITE_QUEUE_NAME, MAPRED_SITE_CONFIG);
- }
- }
- }
- }
-
- /**
* Updates the Spark-related configurations for the clusters managed by this Ambari
* Removes falcon_store_uri from falcon-env.
* Updates {{hdp_full_version}} to {{full_stack_version}}
http://git-wip-us.apache.org/repos/asf/ambari/blob/2b7c97fc/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/webhcat-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/webhcat-site.xml b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/webhcat-site.xml
index d8012dd..cc5de73 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/webhcat-site.xml
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/webhcat-site.xml
@@ -155,6 +155,6 @@ limitations under the License.
<name>yarn.scheduler.capacity.root.queues</name>
</property>
</depends-on>
- <on-ambari-upgrade add="true"/>
+ <on-ambari-upgrade add="false"/>
</property>
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/2b7c97fc/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py
index f8d0c9d..2fc3e8b 100644
--- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py
@@ -112,7 +112,7 @@ oozie_user = config['configurations']['oozie-env']['oozie_user']
smokeuser = config['configurations']['cluster-env']['smokeuser']
smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name']
smoke_hdfs_user_mode = 0770
-service_check_queue_name = default('/configurations/yarn-env/service_check.queue.name','default')
+service_check_queue_name = default('/configurations/yarn-env/service_check.queue.name', 'default')
# This config actually contains {oozie_user}
oozie_admin_users = format(config['configurations']['oozie-env']['oozie_admin_users'])
http://git-wip-us.apache.org/repos/asf/ambari/blob/2b7c97fc/ambari-server/src/main/resources/common-services/SPARK/1.2.1/configuration/spark-defaults.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/configuration/spark-defaults.xml b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/configuration/spark-defaults.xml
index ecb103c..748c270 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/configuration/spark-defaults.xml
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/configuration/spark-defaults.xml
@@ -75,7 +75,7 @@
<name>yarn.scheduler.capacity.root.queues</name>
</property>
</depends-on>
- <on-ambari-upgrade add="true"/>
+ <on-ambari-upgrade add="false"/>
</property>
<property>
<name>spark.yarn.containerLauncherMaxThreads</name>
http://git-wip-us.apache.org/repos/asf/ambari/blob/2b7c97fc/ambari-server/src/main/resources/common-services/SPARK/1.5.2/configuration/spark-thrift-sparkconf.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.5.2/configuration/spark-thrift-sparkconf.xml b/ambari-server/src/main/resources/common-services/SPARK/1.5.2/configuration/spark-thrift-sparkconf.xml
index 865aa73..42ed316 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.5.2/configuration/spark-thrift-sparkconf.xml
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.5.2/configuration/spark-thrift-sparkconf.xml
@@ -91,7 +91,7 @@
<name>yarn.scheduler.capacity.root.queues</name>
</property>
</depends-on>
- <on-ambari-upgrade add="true"/>
+ <on-ambari-upgrade add="false"/>
</property>
<property>
<name>spark.yarn.containerLauncherMaxThreads</name>
http://git-wip-us.apache.org/repos/asf/ambari/blob/2b7c97fc/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/configuration/spark2-defaults.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/configuration/spark2-defaults.xml b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/configuration/spark2-defaults.xml
index 2f9334e..4c28d46 100755
--- a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/configuration/spark2-defaults.xml
+++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/configuration/spark2-defaults.xml
@@ -31,7 +31,7 @@
<name>yarn.scheduler.capacity.root.queues</name>
</property>
</depends-on>
- <on-ambari-upgrade add="true"/>
+ <on-ambari-upgrade add="false"/>
</property>
<property>
<name>spark.driver.extraLibraryPath</name>
http://git-wip-us.apache.org/repos/asf/ambari/blob/2b7c97fc/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/configuration/spark2-thrift-sparkconf.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/configuration/spark2-thrift-sparkconf.xml b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/configuration/spark2-thrift-sparkconf.xml
index dfe87cb..6d73946 100755
--- a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/configuration/spark2-thrift-sparkconf.xml
+++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/configuration/spark2-thrift-sparkconf.xml
@@ -31,7 +31,7 @@
<name>yarn.scheduler.capacity.root.queues</name>
</property>
</depends-on>
- <on-ambari-upgrade add="true"/>
+ <on-ambari-upgrade add="false"/>
</property>
<property>
<name>spark.driver.extraLibraryPath</name>
http://git-wip-us.apache.org/repos/asf/ambari/blob/2b7c97fc/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/configuration/tez-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/configuration/tez-site.xml b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/configuration/tez-site.xml
index e7a851c..a39583c 100644
--- a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/configuration/tez-site.xml
+++ b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/configuration/tez-site.xml
@@ -243,6 +243,6 @@
<name>yarn.scheduler.capacity.root.queues</name>
</property>
</depends-on>
- <on-ambari-upgrade add="true"/>
+ <on-ambari-upgrade add="false"/>
</property>
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/2b7c97fc/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml
index 6951db0..434eea0 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml
@@ -476,6 +476,6 @@
<name>yarn.scheduler.capacity.root.queues</name>
</property>
</depends-on>
- <on-ambari-upgrade add="true"/>
+ <on-ambari-upgrade add="false"/>
</property>
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/2b7c97fc/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-env.xml b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-env.xml
index 152c463..c3bbcb6 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-env.xml
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-env.xml
@@ -255,6 +255,6 @@ YARN_OPTS="$YARN_OPTS -Djava.io.tmpdir={{hadoop_java_io_tmpdir}}"
<name>yarn.scheduler.capacity.root.queues</name>
</property>
</depends-on>
- <on-ambari-upgrade add="true"/>
+ <on-ambari-upgrade add="false"/>
</property>
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/2b7c97fc/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
index b2c635d..4d42861 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
@@ -183,7 +183,7 @@ mapred_log_dir_prefix = config['configurations']['mapred-env']['mapred_log_dir_p
mapred_env_sh_template = config['configurations']['mapred-env']['content']
yarn_env_sh_template = config['configurations']['yarn-env']['content']
yarn_nodemanager_recovery_dir = default('/configurations/yarn-site/yarn.nodemanager.recovery.dir', None)
-service_check_queue_name = default('/configurations/yarn-env/service_check.queue.name','default')
+service_check_queue_name = default('/configurations/yarn-env/service_check.queue.name', 'default')
if len(rm_hosts) > 1:
additional_rm_host = rm_hosts[1]
http://git-wip-us.apache.org/repos/asf/ambari/blob/2b7c97fc/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index 2efb60d..87618a8 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -113,7 +113,11 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
putYarnProperty('yarn.scheduler.minimum-allocation-mb', int(clusterData['ramPerContainer']))
putYarnProperty('yarn.scheduler.maximum-allocation-mb', int(configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"]))
putYarnEnvProperty('min_user_id', self.get_system_min_uid())
- putYarnEnvProperty("service_check.queue.name", self.recommendYarnQueue(services))
+
+ sc_queue_name = self.recommendYarnQueue(services, "yarn-env", "service_check.queue.name")
+ if sc_queue_name is not None:
+ putYarnEnvProperty("service_check.queue.name", sc_queue_name)
+
containerExecutorGroup = 'hadoop'
if 'cluster-env' in services['configurations'] and 'user_group' in services['configurations']['cluster-env']['properties']:
containerExecutorGroup = services['configurations']['cluster-env']['properties']['user_group']
@@ -140,7 +144,9 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
putMapredProperty('mapreduce.map.java.opts', "-Xmx" + str(int(round(0.8 * clusterData['mapMemory']))) + "m")
putMapredProperty('mapreduce.reduce.java.opts', "-Xmx" + str(int(round(0.8 * clusterData['reduceMemory']))) + "m")
putMapredProperty('mapreduce.task.io.sort.mb', min(int(round(0.4 * clusterData['mapMemory'])), 1024))
- putMapredProperty("mapreduce.job.queuename", self.recommendYarnQueue(services))
+ mr_queue = self.recommendYarnQueue(services, "mapred-site", "mapreduce.job.queuename")
+ if mr_queue is not None:
+ putMapredProperty("mapreduce.job.queuename", mr_queue)
def getAmbariUser(self, services):
ambari_user = services['ambari-server-properties']['ambari-server.user']
@@ -1382,34 +1388,36 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
return None
def validatorYarnQueue(self, properties, recommendedDefaults, propertyName, services):
- if not propertyName in properties:
+ if propertyName not in properties:
return self.getErrorItem("Value should be set")
- capacity_scheduler_properties, received_as_key_value_pair = self.getCapacitySchedulerProperties(services)
- leafQueueNames = set()
- leafQueues = self.getAllYarnLeafQueues(capacity_scheduler_properties)
- for queue in leafQueues:
- queue = queue.rstrip('.')
- queueName = queue.split('.')[-1]
- leafQueueNames.add(queueName)
- value = properties[propertyName]
- if len(leafQueueNames) == 0:
+
+ capacity_scheduler_properties, _ = self.getCapacitySchedulerProperties(services)
+ leaf_queue_names = self.getAllYarnLeafQueues(capacity_scheduler_properties)
+ queue_name = properties[propertyName]
+
+ if len(leaf_queue_names) == 0:
return None
- if value not in leafQueueNames:
- return self.getErrorItem("Queue is not exist, or not corresponds to existing YARN leaf queue")
+ elif queue_name not in leaf_queue_names:
+ return self.getErrorItem("Queue is not exist or not corresponds to existing YARN leaf queue")
+
return None
- def recommendYarnQueue(self, services):
- if services:
- if 'configurations' in services:
- capacity_scheduler_properties, received_as_key_value_pair = self.getCapacitySchedulerProperties(services)
- leafQueueNames = set()
- leafQueues = self.getAllYarnLeafQueues(capacity_scheduler_properties)
- for queue in leafQueues:
- queue = queue.rstrip('.')
- queueName = queue.split('.')[-1]
- leafQueueNames.add(queueName)
- if leafQueueNames:
- return leafQueueNames.pop()
+ def recommendYarnQueue(self, services, catalog_name=None, queue_property=None):
+ old_queue_name = None
+
+ if services and 'configurations' in services:
+ configurations = services["configurations"]
+ if catalog_name in configurations and queue_property in configurations[catalog_name]["properties"]:
+ old_queue_name = configurations[catalog_name]["properties"][queue_property]
+
+ capacity_scheduler_properties, _ = self.getCapacitySchedulerProperties(services)
+ leaf_queues = sorted(self.getAllYarnLeafQueues(capacity_scheduler_properties))
+
+ if leaf_queues and (old_queue_name is None or old_queue_name not in leaf_queues):
+ return leaf_queues.pop()
+ elif old_queue_name and old_queue_name in leaf_queues:
+ return None
+
return "default"
def validateXmxValue(self, properties, recommendedDefaults, propertyName):
@@ -1641,7 +1649,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
"""
def getAllYarnLeafQueues(self, capacitySchedulerProperties):
config_list = capacitySchedulerProperties.keys()
- yarn_queues = []
+ yarn_queues = None
leafQueueNames = set()
if 'yarn.scheduler.capacity.root.queues' in config_list:
yarn_queues = capacitySchedulerProperties.get('yarn.scheduler.capacity.root.queues')
http://git-wip-us.apache.org/repos/asf/ambari/blob/2b7c97fc/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
index 6ec4b8d..34b4614 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
@@ -127,7 +127,9 @@ class HDP21StackAdvisor(HDP206StackAdvisor):
putTezProperty("tez.am.java.opts",
"-server -Xmx" + str(int(0.8 * clusterData["amMemory"]))
+ "m -Djava.net.preferIPv4Stack=true -XX:+UseNUMA -XX:+UseParallelGC")
- putTezProperty("tez.queue.name", self.recommendYarnQueue(services))
+ recommended_tez_queue = self.recommendYarnQueue(services, "tez-site", "tez.queue.name")
+ if recommended_tez_queue is not None:
+ putTezProperty("tez.queue.name", recommended_tez_queue)
def getNotPreferableOnServerComponents(self):
http://git-wip-us.apache.org/repos/asf/ambari/blob/2b7c97fc/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
index ea62eb7..a8a75e5 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
@@ -57,12 +57,17 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
:type hosts dict
"""
putSparkProperty = self.putProperty(configurations, "spark-defaults", services)
- putSparkProperty("spark.yarn.queue", self.recommendYarnQueue(services))
+
+ spark_queue = self.recommendYarnQueue(services, "spark-defaults", "spark.yarn.queue")
+ if spark_queue is not None:
+ putSparkProperty("spark.yarn.queue", spark_queue)
# add only if spark supports this config
if "configurations" in services and "spark-thrift-sparkconf" in services["configurations"]:
putSparkThriftSparkConf = self.putProperty(configurations, "spark-thrift-sparkconf", services)
- putSparkThriftSparkConf("spark.yarn.queue", self.recommendYarnQueue(services))
+ recommended_spark_queue = self.recommendYarnQueue(services, "spark-thrift-sparkconf", "spark.yarn.queue")
+ if recommended_spark_queue is not None:
+ putSparkThriftSparkConf("spark.yarn.queue", recommended_spark_queue)
def recommendYARNConfigurations(self, configurations, clusterData, services, hosts):
@@ -445,7 +450,10 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
leafQueues = sorted(leafQueues, key=lambda q:q['value'])
putHiveSitePropertyAttribute("hive.server2.tez.default.queues", "entries", leafQueues)
putHiveSiteProperty("hive.server2.tez.default.queues", ",".join([leafQueue['value'] for leafQueue in leafQueues]))
- putWebhcatSiteProperty("templeton.hadoop.queue.name", self.recommendYarnQueue(services))
+
+ webhcat_queue = self.recommendYarnQueue(services, "webhcat-site", "templeton.hadoop.queue.name")
+ if webhcat_queue is not None:
+ putWebhcatSiteProperty("templeton.hadoop.queue.name", webhcat_queue)
# Recommend Ranger Hive authorization as per Ranger Hive plugin property
@@ -812,7 +820,10 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
putTezProperty("tez.runtime.io.sort.mb", min(int(taskResourceMemory * 0.4), 2047))
putTezProperty("tez.runtime.unordered.output.buffer.size-mb", int(taskResourceMemory * 0.075))
putTezProperty("tez.session.am.dag.submit.timeout.secs", "600")
- putTezProperty("tez.queue.name", self.recommendYarnQueue(services))
+
+ tez_queue = self.recommendYarnQueue(services, "tez-site", "tez.queue.name")
+ if tez_queue is not None:
+ putTezProperty("tez.queue.name", tez_queue)
serverProperties = services["ambari-server-properties"]
latest_tez_jar_version = None
@@ -1079,7 +1090,10 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
putMapredPropertyAttribute("yarn.app.mapreduce.am.resource.mb", "minimum", yarnMinAllocationSize)
# Hadoop MR limitation
putMapredPropertyAttribute("mapreduce.task.io.sort.mb", "maximum", "2047")
- putMapredProperty("mapreduce.job.queuename", self.recommendYarnQueue(services))
+
+ mr_queue = self.recommendYarnQueue(services, "mapred-site", "mapreduce.job.queuename")
+ if mr_queue is not None:
+ putMapredProperty("mapreduce.job.queuename", mr_queue)
def validateMapReduce2Configurations(self, properties, recommendedDefaults, configurations, services, hosts):
validationItems = [ {"config-name": 'mapreduce.map.java.opts', "item": self.validateXmxValue(properties, recommendedDefaults, 'mapreduce.map.java.opts')},
http://git-wip-us.apache.org/repos/asf/ambari/blob/2b7c97fc/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
index 12e68fb..eddccdb 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
@@ -411,8 +411,13 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
putSparkProperty = self.putProperty(configurations, "spark2-defaults", services)
putSparkThriftSparkConf = self.putProperty(configurations, "spark2-thrift-sparkconf", services)
- putSparkProperty("spark.yarn.queue", self.recommendYarnQueue(services))
- putSparkThriftSparkConf("spark.yarn.queue", self.recommendYarnQueue(services))
+ spark_queue = self.recommendYarnQueue(services, "spark2-defaults", "spark.yarn.queue")
+ if spark_queue is not None:
+ putSparkProperty("spark.yarn.queue", spark_queue)
+
+ spart_thrift_queue = self.recommendYarnQueue(services, "spark2-thrift-sparkconf", "spark.yarn.queue")
+ if spart_thrift_queue is not None:
+ putSparkThriftSparkConf("spark.yarn.queue", spart_thrift_queue)
def recommendStormConfigurations(self, configurations, clusterData, services, hosts):
super(HDP25StackAdvisor, self).recommendStormConfigurations(configurations, clusterData, services, hosts)
http://git-wip-us.apache.org/repos/asf/ambari/blob/2b7c97fc/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java
index c680a3b..4c0bc7a 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java
@@ -563,7 +563,6 @@ public class UpgradeCatalog240Test {
Method updateHostRoleCommandTableDML = UpgradeCatalog240.class.getDeclaredMethod("updateHostRoleCommandTableDML");
Method updateKerberosEnv = UpgradeCatalog240.class.getDeclaredMethod("updateKerberosConfigs");
Method updateYarnEnv = UpgradeCatalog240.class.getDeclaredMethod("updateYarnEnv");
- Method updateQueueNameConfigs = UpgradeCatalog240.class.getDeclaredMethod("updateQueueNameConfigs");
Method removeHiveOozieDBConnectionConfigs = UpgradeCatalog240.class.getDeclaredMethod("removeHiveOozieDBConnectionConfigs");
Method updateClustersAndHostsVersionStateTableDML = UpgradeCatalog240.class.getDeclaredMethod("updateClustersAndHostsVersionStateTableDML");
Method removeStandardDeviationAlerts = UpgradeCatalog240.class.getDeclaredMethod("removeStandardDeviationAlerts");
@@ -613,7 +612,6 @@ public class UpgradeCatalog240Test {
.addMockedMethod(updateHostRoleCommandTableDML)
.addMockedMethod(updateKerberosEnv)
.addMockedMethod(updateYarnEnv)
- .addMockedMethod(updateQueueNameConfigs)
.addMockedMethod(removeHiveOozieDBConnectionConfigs)
.addMockedMethod(updateClustersAndHostsVersionStateTableDML)
.addMockedMethod(removeStandardDeviationAlerts)
@@ -654,7 +652,6 @@ public class UpgradeCatalog240Test {
upgradeCatalog240.updateHostRoleCommandTableDML();
upgradeCatalog240.updateKerberosConfigs();
upgradeCatalog240.updateYarnEnv();
- upgradeCatalog240.updateQueueNameConfigs();
upgradeCatalog240.removeHiveOozieDBConnectionConfigs();
upgradeCatalog240.updateClustersAndHostsVersionStateTableDML();
upgradeCatalog240.removeStandardDeviationAlerts();
@@ -1146,156 +1143,6 @@ public class UpgradeCatalog240Test {
assertTrue(Maps.difference(newPropertiesYarnEnv, updatedProperties).areEqual());
}
- /**
- * Test that queue names updated in mapred-site, webhcat-site, tez-site, yarn-env
- * @throws Exception
- */
- @SuppressWarnings("unchecked")
- @Test
- public void testQueueNameUpdateConfigs() throws Exception{
- Map<String, String> oldPropertiesCapacityScheduler = new HashMap<String, String>() {
- {
- put("yarn.scheduler.capacity.root.queues", "default3,d3");
- put("yarn.scheduler.capacity.root.d3.queues", "default4");
- put("yarn.scheduler.capacity.root.default3.queues", "default5");
- }
- };
- Map<String, String> oldPropertiesTezSite = new HashMap<String, String>() {
- {
- put("tez.queue.name", "default3");
- }
- };
- Map<String, String> oldPropertiesWebhcatSite = new HashMap<String, String>() {
- {
- put("templeton.hadoop.queue.name", "default3");
- }
- };
- Map<String, String> oldPropertiesMapredSite = new HashMap<String, String>() {
- {
- put("mapreduce.job.queuename", "default3");
- }
- };
- Map<String, String> oldPropertiesYarnEnv = new HashMap<String, String>() {
- {
- put("service_check.queue.name", "default3");
- }
- };
-
- Map<String, String> newPropertiesTezSite = new HashMap<String, String>() {
- {
- put("tez.queue.name", "default5");
- }
- };
- Map<String, String> newPropertiesWebhcatSite = new HashMap<String, String>() {
- {
- put("templeton.hadoop.queue.name", "default5");
- }
- };
- Map<String, String> newPropertiesMapredSite = new HashMap<String, String>() {
- {
- put("mapreduce.job.queuename", "default5");
- }
- };
- Map<String, String> newPropertiesYarnEnv = new HashMap<String, String>() {
- {
- put("service_check.queue.name", "default5");
- }
- };
-
- EasyMockSupport easyMockSupport = new EasyMockSupport();
-
- Clusters clusters = easyMockSupport.createNiceMock(Clusters.class);
- final Cluster cluster = easyMockSupport.createNiceMock(Cluster.class);
- Config mockCapacityScheduler = easyMockSupport.createNiceMock(Config.class);
- Config mockWebhcatSite = easyMockSupport.createNiceMock(Config.class);
- Config mockTezSite = easyMockSupport.createNiceMock(Config.class);
- Config mockMapredSite = easyMockSupport.createNiceMock(Config.class);
- Config mockYarnEnv = easyMockSupport.createNiceMock(Config.class);
-
- expect(clusters.getClusters()).andReturn(new HashMap<String, Cluster>() {{
- put("normal", cluster);
- }}).anyTimes();
-
- expect(cluster.getServices()).andReturn(new HashMap<String, Service>() {
- {
- put("YARN", null);
- put("HIVE", null);
- put("MAPREDUCE", null);
- put("TEZ", null);
- }
- }).atLeastOnce();
-
- expect(cluster.getServiceByConfigType(WEBHCAT_SITE_CONFIG_TYPE)).andReturn("HIVE").atLeastOnce();
- expect(cluster.getServiceByConfigType(TEZ_SITE_CONFIG_TYPE)).andReturn("TEZ").atLeastOnce();
- expect(cluster.getServiceByConfigType(MAPRED_SITE_CONFIG_TYPE)).andReturn("MAPREDUCE").atLeastOnce();
- expect(cluster.getServiceByConfigType(YARN_ENV_CONFIG_TYPE)).andReturn("YARN").atLeastOnce();
-
- expect(cluster.getDesiredConfigByType(CAPACITY_SCHEDULER_CONFIG_TYPE)).andReturn(mockCapacityScheduler).atLeastOnce();
- expect(cluster.getDesiredConfigByType(WEBHCAT_SITE_CONFIG_TYPE)).andReturn(mockWebhcatSite).atLeastOnce();
- expect(cluster.getDesiredConfigByType(TEZ_SITE_CONFIG_TYPE)).andReturn(mockTezSite).atLeastOnce();
- expect(cluster.getDesiredConfigByType(MAPRED_SITE_CONFIG_TYPE)).andReturn(mockMapredSite).atLeastOnce();
- expect(cluster.getDesiredConfigByType(YARN_ENV_CONFIG_TYPE)).andReturn(mockYarnEnv).atLeastOnce();
-
-
-
- expect(mockCapacityScheduler.getProperties()).andReturn(oldPropertiesCapacityScheduler).anyTimes();
- expect(mockWebhcatSite.getProperties()).andReturn(oldPropertiesWebhcatSite).anyTimes();
- expect(mockTezSite.getProperties()).andReturn(oldPropertiesTezSite).anyTimes();
- expect(mockMapredSite.getProperties()).andReturn(oldPropertiesMapredSite).anyTimes();
- expect(mockYarnEnv.getProperties()).andReturn(oldPropertiesYarnEnv).anyTimes();
-
- Injector injector = easyMockSupport.createNiceMock(Injector.class);
- expect(injector.getInstance(Gson.class)).andReturn(null).anyTimes();
- expect(injector.getInstance(MaintenanceStateHelper.class)).andReturn(null).anyTimes();
- expect(injector.getInstance(KerberosHelper.class)).andReturn(createNiceMock(KerberosHelper.class)).anyTimes();
-
- replay(injector, clusters, mockCapacityScheduler, mockWebhcatSite, mockTezSite, mockMapredSite,
- mockYarnEnv, cluster);
-
- AmbariManagementControllerImpl controller = createMockBuilder(AmbariManagementControllerImpl.class)
- .addMockedMethod("createConfiguration")
- .addMockedMethod("getClusters", new Class[] { })
- .addMockedMethod("createConfig")
- .withConstructor(createNiceMock(ActionManager.class), clusters, injector)
- .createNiceMock();
-
- Injector injector2 = easyMockSupport.createNiceMock(Injector.class);
- Capture<Map> propertiesCapacityScheduler = EasyMock.newCapture();
- Capture<Map> propertiesCaptureYarnEnv = EasyMock.newCapture();
- Capture<Map> propertiesCaptureWebhcatSite = EasyMock.newCapture();
- Capture<Map> propertiesCaptureTezSite= EasyMock.newCapture();
- Capture<Map> propertiesCaptureMapredSite = EasyMock.newCapture();
-
- expect(injector2.getInstance(AmbariManagementController.class)).andReturn(controller).anyTimes();
- expect(controller.getClusters()).andReturn(clusters).anyTimes();
- expect(controller.createConfig(anyObject(Cluster.class), eq(CAPACITY_SCHEDULER_CONFIG_TYPE), capture(propertiesCapacityScheduler), anyString(),
- anyObject(Map.class))).andReturn(createNiceMock(Config.class)).once();
- expect(controller.createConfig(anyObject(Cluster.class), eq(YARN_ENV_CONFIG_TYPE), capture(propertiesCaptureYarnEnv), anyString(),
- anyObject(Map.class))).andReturn(createNiceMock(Config.class)).once();
- expect(controller.createConfig(anyObject(Cluster.class), eq(WEBHCAT_SITE_CONFIG_TYPE), capture(propertiesCaptureWebhcatSite), anyString(),
- anyObject(Map.class))).andReturn(createNiceMock(Config.class)).once();
- expect(controller.createConfig(anyObject(Cluster.class), eq(TEZ_SITE_CONFIG_TYPE), capture(propertiesCaptureTezSite), anyString(),
- anyObject(Map.class))).andReturn(createNiceMock(Config.class)).once();
- expect(controller.createConfig(anyObject(Cluster.class), eq(MAPRED_SITE_CONFIG_TYPE), capture(propertiesCaptureMapredSite), anyString(),
- anyObject(Map.class))).andReturn(createNiceMock(Config.class)).once();
-
- replay(controller, injector2);
- new UpgradeCatalog240(injector2).updateQueueNameConfigs();
- easyMockSupport.verifyAll();
-
- Map<String, String> updatedPropertiesYarnEnv = propertiesCaptureYarnEnv.getValue();
- assertTrue(Maps.difference(newPropertiesYarnEnv, updatedPropertiesYarnEnv).areEqual());
-
- Map<String, String> updatedPropertiesWebhcatSite = propertiesCaptureWebhcatSite.getValue();
- assertTrue(Maps.difference(newPropertiesWebhcatSite, updatedPropertiesWebhcatSite).areEqual());
-
- Map<String, String> updatedPropertiesTezSite = propertiesCaptureTezSite.getValue();
- assertTrue(Maps.difference(newPropertiesTezSite, updatedPropertiesTezSite).areEqual());
-
- Map<String, String> updatedPropertiesMapredSite = propertiesCaptureMapredSite.getValue();
- assertTrue(Maps.difference(newPropertiesMapredSite, updatedPropertiesMapredSite).areEqual());
- }
-
@Test
public void testSparkConfigUpdate() throws Exception{
http://git-wip-us.apache.org/repos/asf/ambari/blob/2b7c97fc/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
index 4a12920..e28389e 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
@@ -222,7 +222,7 @@ class TestHDP206StackAdvisor(TestCase):
hosts = self.prepareHosts([])
result = self.stackAdvisor.validateConfigurations(services, hosts)
expectedItems = [
- {'message': 'Queue is not exist, or not corresponds to existing YARN leaf queue', 'level': 'ERROR'}
+ {'message': 'Queue is not exist or not corresponds to existing YARN leaf queue', 'level': 'ERROR'}
]
self.assertValidationResult(expectedItems, result)
services["configurations"]["yarn-env"]["properties"]["service_check.queue.name"] = "ndfqueue"
http://git-wip-us.apache.org/repos/asf/ambari/blob/2b7c97fc/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
index 29a47d9..3f41acb 100644
--- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
@@ -1283,7 +1283,7 @@ class TestHDP22StackAdvisor(TestCase):
},
'webhcat-site': {
'properties': {
- 'templeton.hadoop.queue.name': 'queue1'
+ 'templeton.hadoop.queue.name': 'queue2'
}
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/2b7c97fc/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
index 272269d..137fd47 100644
--- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
@@ -657,7 +657,7 @@ class TestHDP23StackAdvisor(TestCase):
},
'webhcat-site': {
'properties': {
- 'templeton.hadoop.queue.name': 'queue1'
+ 'templeton.hadoop.queue.name': 'queue2'
}
}
}
@@ -933,7 +933,7 @@ class TestHDP23StackAdvisor(TestCase):
},
'webhcat-site': {
'properties': {
- 'templeton.hadoop.queue.name': 'queue1'
+ 'templeton.hadoop.queue.name': 'queue2'
}
}
}
@@ -1163,7 +1163,7 @@ class TestHDP23StackAdvisor(TestCase):
"tez.session.am.dag.submit.timeout.secs": "600",
"tez.runtime.unordered.output.buffer.size-mb": "57",
"tez.am.resource.memory.mb": "4000",
- "tez.queue.name": "queue1",
+ "tez.queue.name": "queue2",
}
},
"yarn-site": {
http://git-wip-us.apache.org/repos/asf/ambari/blob/2b7c97fc/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
index d1f6d88..0ed1761 100644
--- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
@@ -7252,7 +7252,7 @@ class TestHDP25StackAdvisor(TestCase):
hosts = self.prepareHosts([])
result = self.stackAdvisor.validateConfigurations(services, hosts)
expectedItems = [
- {'message': 'Queue is not exist, or not corresponds to existing YARN leaf queue', 'level': 'ERROR'}
+ {'message': 'Queue is not exist or not corresponds to existing YARN leaf queue', 'level': 'ERROR'}
]
self.assertValidationResult(expectedItems, result)
services["configurations"]["yarn-env"]["properties"]["service_check.queue.name"] = "ndfqueue2"