You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by dm...@apache.org on 2016/06/21 15:00:50 UTC
[2/2] ambari git commit: AMBARI-17200. Tez, Pig, Hive,
MapReduce service checks will fail on non standard queue names
(dgrinenko via dlysnichenko)
AMBARI-17200. Tez, Pig, Hive, MapReduce service checks will fail on non standard queue names (dgrinenko via dlysnichenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/16175189
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/16175189
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/16175189
Branch: refs/heads/branch-2.4
Commit: 16175189b0818734ef29861e8253a249e155c084
Parents: 92c8a26
Author: Lisnichenko Dmitro <dl...@hortonworks.com>
Authored: Tue Jun 21 17:59:44 2016 +0300
Committer: Lisnichenko Dmitro <dl...@hortonworks.com>
Committed: Tue Jun 21 18:00:40 2016 +0300
----------------------------------------------------------------------
.../server/upgrade/AbstractUpgradeCatalog.java | 84 +++++++-
.../server/upgrade/UpgradeCatalog240.java | 52 +++++
.../0.12.0.2.0/configuration/webhcat-site.xml | 14 ++
.../TEZ/0.4.0.2.1/configuration/tez-site.xml | 14 ++
.../configuration-mapred/mapred-site.xml | 14 ++
.../YARN/2.1.0.2.0/configuration/yarn-env.xml | 6 +
.../stacks/HDP/2.0.6/services/stack_advisor.py | 35 +++-
.../stacks/HDP/2.1/services/stack_advisor.py | 4 +-
.../stacks/HDP/2.2/services/stack_advisor.py | 18 +-
.../server/upgrade/UpgradeCatalog240Test.java | 200 +++++++++++++++++--
.../stacks/2.0.6/common/test_stack_advisor.py | 53 +----
.../stacks/2.2/common/test_stack_advisor.py | 55 +++--
.../stacks/2.3/common/test_stack_advisor.py | 13 +-
.../stacks/2.5/common/test_stack_advisor.py | 22 +-
14 files changed, 471 insertions(+), 113 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/16175189/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java
index 7856969..bff2305 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java
@@ -17,10 +17,6 @@
*/
package org.apache.ambari.server.upgrade;
-import javax.persistence.EntityManager;
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-
import java.io.StringReader;
import java.sql.ResultSet;
import java.sql.SQLException;
@@ -34,8 +30,14 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
+import java.util.Stack;
+import java.util.StringTokenizer;
import java.util.TreeMap;
+import javax.persistence.EntityManager;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.api.services.AmbariMetaInfo;
import org.apache.ambari.server.configuration.Configuration;
@@ -115,6 +117,10 @@ public abstract class AbstractUpgradeCatalog implements UpgradeCatalog {
public static final String PROPERTY_RANGER_HBASE_PLUGIN_ENABLED = "ranger-hbase-plugin-enabled";
public static final String PROPERTY_RANGER_KNOX_PLUGIN_ENABLED = "ranger-knox-plugin-enabled";
+ public static final String YARN_SCHEDULER_CAPACITY_ROOT_QUEUE = "yarn.scheduler.capacity.root";
+ public static final String YARN_SCHEDULER_CAPACITY_ROOT_QUEUES = "yarn.scheduler.capacity.root.queues";
+ public static final String QUEUES = "queues";
+
public static final String ALERT_URL_PROPERTY_CONNECTION_TIMEOUT = "connection_timeout";
private static final Logger LOG = LoggerFactory.getLogger
@@ -913,4 +919,74 @@ public abstract class AbstractUpgradeCatalog implements UpgradeCatalog {
public void onPostUpgrade() throws AmbariException, SQLException {
// NOOP
}
+
+ /**
+ * Validate queueNameProperty exists for configType in cluster and corresponds to one of validLeafQueues
+ * @param cluster cluster to operate with
+ * @param validLeafQueues Set of YARN capacity-scheduler leaf queues
+ * @param queueNameProperty queue name property to check and update
+ * @param configType config type name
+ * @return
+ */
+ protected boolean isQueueNameValid(Cluster cluster, Set<String> validLeafQueues, String queueNameProperty, String configType) {
+ Config site = cluster.getDesiredConfigByType(configType);
+ Map<String, String> properties = site.getProperties();
+ boolean result = properties.containsKey(queueNameProperty) && validLeafQueues.contains(properties.get(queueNameProperty));
+ if (!result){
+ LOG.info("Queue name " + queueNameProperty + " in " + configType + " not defined or not corresponds to valid capacity-scheduler queue");
+ }
+ return result;
+ }
+
+
+ /**
+ * Update property queueNameProperty from configType of cluster to first of validLeafQueues
+ * @param cluster cluster to operate with
+ * @param validLeafQueues Set of YARN capacity-scheduler leaf queues
+ * @param queueNameProperty queue name property to check and update
+ * @param configType config type name
+ * @throws AmbariException if an error occurs while updating the configurations
+ */
+ protected void updateQueueName(Cluster cluster, Set<String> validLeafQueues, String queueNameProperty, String configType) throws AmbariException {
+ String recommendQueue = validLeafQueues.iterator().next();
+ LOG.info("Update " + queueNameProperty + " in " + configType + " set to " + recommendQueue);
+ Map<String, String> updates = Collections.singletonMap(queueNameProperty, recommendQueue);
+ updateConfigurationPropertiesForCluster(cluster, configType, updates, true, true);
+ }
+
+ /**
+ * Pars Capacity Scheduler Properties and get all YARN Capacity Scheduler leaf queue names
+ * @param capacitySchedulerMap capacity-scheduler properties map
+ * @return all YARN Capacity Scheduler leaf queue names
+ */
+ protected Set<String> getCapacitySchedulerLeafQueues(Map<String, String> capacitySchedulerMap) {
+ Set<String> leafQueues= new HashSet<>();
+ Stack<String> toProcessQueues = new Stack<>();
+ if (capacitySchedulerMap.containsKey(YARN_SCHEDULER_CAPACITY_ROOT_QUEUES)){
+ StringTokenizer queueTokenizer = new StringTokenizer(capacitySchedulerMap.get(
+ YARN_SCHEDULER_CAPACITY_ROOT_QUEUES), ",");
+ while (queueTokenizer.hasMoreTokens()){
+ toProcessQueues.push(queueTokenizer.nextToken());
+ }
+ }
+ while (!toProcessQueues.empty()){
+ String queue = toProcessQueues.pop();
+ String queueKey = YARN_SCHEDULER_CAPACITY_ROOT_QUEUE + "." + queue + "." + QUEUES;
+ if (capacitySchedulerMap.containsKey(queueKey)){
+ StringTokenizer queueTokenizer = new StringTokenizer(capacitySchedulerMap.get(queueKey), ",");
+ while (queueTokenizer.hasMoreTokens()){
+ toProcessQueues.push(queue + "." + queueTokenizer.nextToken());
+ }
+ } else {
+ if (!queue.endsWith(".")){
+ String queueName = queue.substring(queue.lastIndexOf('.')+1);
+ leafQueues.add(queueName);
+ } else {
+ LOG.warn("Queue " + queue + " is not valid");
+ }
+ }
+ }
+ return leafQueues;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/16175189/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java
index 357b776..72f5052 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java
@@ -144,6 +144,10 @@ public class UpgradeCatalog240 extends AbstractUpgradeCatalog {
protected static final String CLUSTER_TABLE = "clusters";
protected static final String CLUSTER_UPGRADE_ID_COLUMN = "upgrade_id";
protected static final String YARN_ENV_CONFIG = "yarn-env";
+ protected static final String CAPACITY_SCHEDULER_CONFIG = "capacity-scheduler";
+ protected static final String WEBHCAT_SITE_CONFIG = "webhcat-site";
+ protected static final String TEZ_SITE_CONFIG = "tez-site";
+ protected static final String MAPRED_SITE_CONFIG = "mapred-site";
public static final String DESIRED_VERSION_COLUMN_NAME = "desired_version";
public static final String BLUEPRINT_SETTING_TABLE = "blueprint_setting";
public static final String BLUEPRINT_NAME_COL = "blueprint_name";
@@ -182,6 +186,10 @@ public class UpgradeCatalog240 extends AbstractUpgradeCatalog {
protected static final String EXTENSION_LINK_ID_COLUMN = "link_id";
private static final Map<String, Integer> ROLE_ORDER;
+ public static final String WEBHCAT_SITE_QUEUE_NAME = "templeton.hadoop.queue.name";
+ public static final String TEZ_SITE_QUEUE_NAME = "tez.queue.name";
+ public static final String YARN_ENV_QUEUE_NAME = "service_check.queue.name";
+ public static final String MAPRED_SITE_QUEUE_NAME = "mapreduce.job.queuename";
static {
// Manually create role order since there really isn't any mechanism for this
@@ -375,6 +383,7 @@ public class UpgradeCatalog240 extends AbstractUpgradeCatalog {
updateSparkConfigs();
updateHBaseConfigs();
updateFalconConfigs();
+ updateQueueNameConfigs();
updateKerberosDescriptorArtifacts();
removeHiveOozieDBConnectionConfigs();
updateClustersAndHostsVersionStateTableDML();
@@ -1964,6 +1973,49 @@ public class UpgradeCatalog240 extends AbstractUpgradeCatalog {
}
/**
+ * Updates the YARN Capacity Scheduler related configurations for the clusters managed by this Ambari
+ * Update services configuration to set proper leaf queue name for services
+ * update mapred-site, webhcat-site, tez-site, yarn-env
+ *
+ * @throws AmbariException if an error occurs while updating the configurations
+ */
+ protected void updateQueueNameConfigs() throws AmbariException {
+ AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
+ Clusters clusters = ambariManagementController.getClusters();
+ Map<String, Cluster> clusterMap = getCheckedClusterMap(clusters);
+
+ for (final Cluster cluster : clusterMap.values()) {
+ Config capacityScheduler = cluster.getDesiredConfigByType(CAPACITY_SCHEDULER_CONFIG);
+ if (capacityScheduler != null) {
+ Map<String, String> capacitySchedulerProperties = capacityScheduler.getProperties();
+ Set<String> leafQueues;
+ leafQueues = getCapacitySchedulerLeafQueues(capacitySchedulerProperties);
+ Set<String> installedServices = cluster.getServices().keySet();
+ if (leafQueues ==null || leafQueues.isEmpty()) {
+ LOG.warn("There is no leafQueues in capacity-scheduler");
+ return;
+ }
+ if (installedServices.contains(cluster.getServiceByConfigType(WEBHCAT_SITE_CONFIG)) &&
+ !isQueueNameValid(cluster, leafQueues, WEBHCAT_SITE_QUEUE_NAME, WEBHCAT_SITE_CONFIG)){
+ updateQueueName(cluster, leafQueues, WEBHCAT_SITE_QUEUE_NAME, WEBHCAT_SITE_CONFIG);
+ }
+ if (installedServices.contains(cluster.getServiceByConfigType(TEZ_SITE_CONFIG)) &&
+ !isQueueNameValid(cluster, leafQueues, TEZ_SITE_QUEUE_NAME, TEZ_SITE_CONFIG)){
+ updateQueueName(cluster, leafQueues, TEZ_SITE_QUEUE_NAME, TEZ_SITE_CONFIG);
+ }
+ if (installedServices.contains(cluster.getServiceByConfigType(YARN_ENV_CONFIG)) &&
+ !isQueueNameValid(cluster, leafQueues, YARN_ENV_QUEUE_NAME, YARN_ENV_CONFIG)){
+ updateQueueName(cluster, leafQueues, YARN_ENV_QUEUE_NAME, YARN_ENV_CONFIG);
+ }
+ if (installedServices.contains(cluster.getServiceByConfigType(MAPRED_SITE_CONFIG)) &&
+ !isQueueNameValid(cluster, leafQueues, MAPRED_SITE_QUEUE_NAME, MAPRED_SITE_CONFIG)){
+ updateQueueName(cluster, leafQueues, MAPRED_SITE_QUEUE_NAME, MAPRED_SITE_CONFIG);
+ }
+ }
+ }
+ }
+
+ /**
* Updates the Spark-related configurations for the clusters managed by this Ambari
* Removes falcon_store_uri from falcon-env.
* Updates {{hdp_full_version}} to {{full_stack_version}}
http://git-wip-us.apache.org/repos/asf/ambari/blob/16175189/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/webhcat-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/webhcat-site.xml b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/webhcat-site.xml
index eaa5334..d8012dd 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/webhcat-site.xml
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/webhcat-site.xml
@@ -143,4 +143,18 @@ limitations under the License.
<description>Time out for templeton api</description>
<on-ambari-upgrade add="true"/>
</property>
+ <property>
+ <name>templeton.hadoop.queue.name</name>
+ <value>default</value>
+ <description>
+ MapReduce queue name where WebHCat map-only jobs will be submitted to. Can be used to avoid a deadlock where all map slots in the cluster are taken over by Templeton launcher tasks.
+ </description>
+ <depends-on>
+ <property>
+ <type>capacity-scheduler</type>
+ <name>yarn.scheduler.capacity.root.queues</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="true"/>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/16175189/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/configuration/tez-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/configuration/tez-site.xml b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/configuration/tez-site.xml
index 9d54dd6..e7a851c 100644
--- a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/configuration/tez-site.xml
+++ b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/configuration/tez-site.xml
@@ -231,4 +231,18 @@
<description>Whether to send history events to YARN Application Timeline Server</description>
<on-ambari-upgrade add="true"/>
</property>
+ <property>
+ <name>tez.queue.name</name>
+ <value>default</value>
+ <description>
+ Specify which queue will be used for Tez jobs.
+ </description>
+ <depends-on>
+ <property>
+ <type>capacity-scheduler</type>
+ <name>yarn.scheduler.capacity.root.queues</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="true"/>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/16175189/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml
index 0d6c617..6951db0 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml
@@ -464,4 +464,18 @@
</description>
<on-ambari-upgrade add="true"/>
</property>
+ <property>
+ <name>mapreduce.job.queuename</name>
+ <value>default</value>
+ <description>
+ Queue to which a job is submitted.
+ </description>
+ <depends-on>
+ <property>
+ <type>capacity-scheduler</type>
+ <name>yarn.scheduler.capacity.root.queues</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="true"/>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/16175189/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-env.xml b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-env.xml
index b035ea3..152c463 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-env.xml
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-env.xml
@@ -249,6 +249,12 @@ YARN_OPTS="$YARN_OPTS -Djava.io.tmpdir={{hadoop_java_io_tmpdir}}"
<description>
The queue that used by service check.
</description>
+ <depends-on>
+ <property>
+ <type>capacity-scheduler</type>
+ <name>yarn.scheduler.capacity.root.queues</name>
+ </property>
+ </depends-on>
<on-ambari-upgrade add="true"/>
</property>
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/16175189/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index dd03a9f..373403c 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -112,6 +112,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
putYarnProperty('yarn.scheduler.minimum-allocation-mb', int(clusterData['ramPerContainer']))
putYarnProperty('yarn.scheduler.maximum-allocation-mb', int(configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"]))
putYarnEnvProperty('min_user_id', self.get_system_min_uid())
+ putYarnEnvProperty("service_check.queue.name", self.recommendYarnQueue(services))
containerExecutorGroup = 'hadoop'
if 'cluster-env' in services['configurations'] and 'user_group' in services['configurations']['cluster-env']['properties']:
containerExecutorGroup = services['configurations']['cluster-env']['properties']['user_group']
@@ -138,6 +139,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
putMapredProperty('mapreduce.map.java.opts', "-Xmx" + str(int(round(0.8 * clusterData['mapMemory']))) + "m")
putMapredProperty('mapreduce.reduce.java.opts', "-Xmx" + str(int(round(0.8 * clusterData['reduceMemory']))) + "m")
putMapredProperty('mapreduce.task.io.sort.mb', min(int(round(0.4 * clusterData['mapMemory'])), 1024))
+ putMapredProperty("mapreduce.job.queuename", self.recommendYarnQueue(services))
def getAmbariUser(self, services):
ambari_user = services['ambari-server-properties']['ambari-server.user']
@@ -1354,6 +1356,26 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
return None
+ def validatorYarnQueue(self, properties, recommendedDefaults, propertyName, services):
+ if not propertyName in properties:
+ return self.getErrorItem("Value should be set")
+ capacity_scheduler_properties, received_as_key_value_pair = self.getCapacitySchedulerProperties(services)
+ leafQueueNames = self.getAllYarnLeafQueues(capacity_scheduler_properties)
+ value = properties[propertyName]
+ if len(leafQueueNames) == 0:
+ return None
+ if value not in leafQueueNames:
+ return self.getErrorItem("Queue is not exist, or not corresponds to existing YARN leaf queue")
+ return None
+
+ def recommendYarnQueue(self, services):
+ if services:
+ if 'configurations' in services:
+ capacity_scheduler_properties, received_as_key_value_pair = self.getCapacitySchedulerProperties(services)
+ leafQueueNames = self.getAllYarnLeafQueues(capacity_scheduler_properties)
+ if leafQueueNames:
+ return leafQueueNames.pop()
+ return "default"
def validateXmxValue(self, properties, recommendedDefaults, propertyName):
if not propertyName in properties:
@@ -1382,7 +1404,8 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
{"config-name": 'mapreduce.map.memory.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'mapreduce.map.memory.mb')},
{"config-name": 'mapreduce.reduce.memory.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'mapreduce.reduce.memory.mb')},
{"config-name": 'yarn.app.mapreduce.am.resource.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.app.mapreduce.am.resource.mb')},
- {"config-name": 'yarn.app.mapreduce.am.command-opts', "item": self.validateXmxValue(properties, recommendedDefaults, 'yarn.app.mapreduce.am.command-opts')} ]
+ {"config-name": 'yarn.app.mapreduce.am.command-opts', "item": self.validateXmxValue(properties, recommendedDefaults, 'yarn.app.mapreduce.am.command-opts')},
+ {"config-name": 'mapreduce.job.queuename', "item": self.validatorYarnQueue(properties, recommendedDefaults, 'mapreduce.job.queuename', services)} ]
return self.toConfigurationValidationProblems(validationItems, "mapred-site")
def validateYARNConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
@@ -1394,15 +1417,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
return self.toConfigurationValidationProblems(validationItems, "yarn-site")
def validateYARNEnvConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
- validationItems = [ ]
- if not services:
- return self.toConfigurationValidationProblems(validationItems, "yarn-env")
- yarnEnvProperties = getSiteProperties(configurations, "yarn-env")
- capacity_scheduler_properties, received_as_key_value_pair = self.getCapacitySchedulerProperties(services)
- leafQueueNames = self.getAllYarnLeafQueues(capacity_scheduler_properties)
- service_checkQueueName=yarnEnvProperties.get("service_check.queue.name")
- if service_checkQueueName not in leafQueueNames:
- validationItems.append({"config-name": 'service_check.queue.name', "item": self.getErrorItem("service_check.queue.name is not exist, or not corresponds to existing leaf queue")})
+ validationItems = [{"config-name": 'service_check.queue.name', "item": self.validatorYarnQueue(properties, recommendedDefaults, 'service_check.queue.name', services)} ]
return self.toConfigurationValidationProblems(validationItems, "yarn-env")
def validateHbaseEnvConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
http://git-wip-us.apache.org/repos/asf/ambari/blob/16175189/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
index f63ca86..3d6ca4c 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
@@ -113,6 +113,7 @@ class HDP21StackAdvisor(HDP206StackAdvisor):
putTezProperty("tez.am.java.opts",
"-server -Xmx" + str(int(0.8 * clusterData["amMemory"]))
+ "m -Djava.net.preferIPv4Stack=true -XX:+UseNUMA -XX:+UseParallelGC")
+ putTezProperty("tez.queue.name", self.recommendYarnQueue(services))
def getNotPreferableOnServerComponents(self):
@@ -153,7 +154,8 @@ class HDP21StackAdvisor(HDP206StackAdvisor):
def validateTezConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
validationItems = [ {"config-name": 'tez.am.resource.memory.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'tez.am.resource.memory.mb')},
- {"config-name": 'tez.am.java.opts', "item": self.validateXmxValue(properties, recommendedDefaults, 'tez.am.java.opts')} ]
+ {"config-name": 'tez.am.java.opts', "item": self.validateXmxValue(properties, recommendedDefaults, 'tez.am.java.opts')},
+ {"config-name": 'tez.queue.name', "item": self.validatorYarnQueue(properties, recommendedDefaults, 'tez.queue.name', services)} ]
return self.toConfigurationValidationProblems(validationItems, "tez-site")
def getDBDriver(self, databaseType):
http://git-wip-us.apache.org/repos/asf/ambari/blob/16175189/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
index 85f679b..38586e4 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
@@ -267,6 +267,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
putHiveServerProperty = self.putProperty(configurations, "hiveserver2-site", services)
putHiveEnvProperty = self.putProperty(configurations, "hive-env", services)
putHiveSiteProperty = self.putProperty(configurations, "hive-site", services)
+ putWebhcatSiteProperty = self.putProperty(configurations, "webhcat-site", services)
putHiveSitePropertyAttribute = self.putPropertyAttribute(configurations, "hive-site")
putHiveEnvPropertyAttributes = self.putPropertyAttribute(configurations, "hive-env")
servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
@@ -405,6 +406,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
leafQueues = sorted(leafQueues, key=lambda q:q['value'])
putHiveSitePropertyAttribute("hive.server2.tez.default.queues", "entries", leafQueues)
putHiveSiteProperty("hive.server2.tez.default.queues", ",".join([leafQueue['value'] for leafQueue in leafQueues]))
+ putWebhcatSiteProperty("templeton.hadoop.queue.name", self.recommendYarnQueue(services))
# Recommend Ranger Hive authorization as per Ranger Hive plugin property
@@ -769,6 +771,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
putTezProperty("tez.runtime.io.sort.mb", min(int(taskResourceMemory * 0.4), 2047))
putTezProperty("tez.runtime.unordered.output.buffer.size-mb", int(taskResourceMemory * 0.075))
putTezProperty("tez.session.am.dag.submit.timeout.secs", "600")
+ putTezProperty("tez.queue.name", self.recommendYarnQueue(services))
serverProperties = services["ambari-server-properties"]
latest_tez_jar_version = None
@@ -934,7 +937,8 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
"ranger-yarn-plugin-properties": self.validateYARNRangerPluginConfigurations},
"HIVE": {"hiveserver2-site": self.validateHiveServer2Configurations,
"hive-site": self.validateHiveConfigurations,
- "hive-env": self.validateHiveConfigurationsEnv},
+ "hive-env": self.validateHiveConfigurationsEnv,
+ "webhcat-site": self.validateWebhcatConfigurations},
"HBASE": {"hbase-site": self.validateHBASEConfigurations,
"hbase-env": self.validateHBASEEnvConfigurations,
"ranger-hbase-plugin-properties": self.validateHBASERangerPluginConfigurations},
@@ -976,7 +980,8 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
validationItems = [ {"config-name": 'tez.am.resource.memory.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'tez.am.resource.memory.mb')},
{"config-name": 'tez.task.resource.memory.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'tez.task.resource.memory.mb')},
{"config-name": 'tez.runtime.io.sort.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'tez.runtime.io.sort.mb')},
- {"config-name": 'tez.runtime.unordered.output.buffer.size-mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'tez.runtime.unordered.output.buffer.size-mb')},]
+ {"config-name": 'tez.runtime.unordered.output.buffer.size-mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'tez.runtime.unordered.output.buffer.size-mb')},
+ {"config-name": 'tez.queue.name', "item": self.validatorYarnQueue(properties, recommendedDefaults, 'tez.queue.name', services)} ]
if "tez.tez-ui.history-url.base" in recommendedDefaults:
validationItems.append({"config-name": 'tez.tez-ui.history-url.base', "item": self.validatorEqualsToRecommendedItem(properties, recommendedDefaults, 'tez.tez-ui.history-url.base')})
@@ -1031,6 +1036,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
putMapredPropertyAttribute("yarn.app.mapreduce.am.resource.mb", "minimum", yarnMinAllocationSize)
# Hadoop MR limitation
putMapredPropertyAttribute("mapreduce.task.io.sort.mb", "maximum", "2047")
+ putMapredProperty("mapreduce.job.queuename", self.recommendYarnQueue(services))
def validateMapReduce2Configurations(self, properties, recommendedDefaults, configurations, services, hosts):
validationItems = [ {"config-name": 'mapreduce.map.java.opts', "item": self.validateXmxValue(properties, recommendedDefaults, 'mapreduce.map.java.opts')},
@@ -1039,7 +1045,8 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
{"config-name": 'mapreduce.map.memory.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'mapreduce.map.memory.mb')},
{"config-name": 'mapreduce.reduce.memory.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'mapreduce.reduce.memory.mb')},
{"config-name": 'yarn.app.mapreduce.am.resource.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.app.mapreduce.am.resource.mb')},
- {"config-name": 'yarn.app.mapreduce.am.command-opts', "item": self.validateXmxValue(properties, recommendedDefaults, 'yarn.app.mapreduce.am.command-opts')}]
+ {"config-name": 'yarn.app.mapreduce.am.command-opts', "item": self.validateXmxValue(properties, recommendedDefaults, 'yarn.app.mapreduce.am.command-opts')},
+ {"config-name": 'mapreduce.job.queuename', "item": self.validatorYarnQueue(properties, recommendedDefaults, 'mapreduce.job.queuename', services)} ]
if 'mapreduce.map.java.opts' in properties and \
checkXmxValueFormat(properties['mapreduce.map.java.opts']):
@@ -1295,6 +1302,11 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
" {0} needs to be set to {1}".format(prop_name,prop_val))})
return self.toConfigurationValidationProblems(validationItems, "hiveserver2-site")
+ def validateWebhcatConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
+ validationItems = [{"config-name": 'templeton.hadoop.queue.name', "item": self.validatorYarnQueue(properties, recommendedDefaults, 'templeton.hadoop.queue.name', services)}]
+ return self.toConfigurationValidationProblems(validationItems, "webhcat-site")
+
+
def validateHiveConfigurationsEnv(self, properties, recommendedDefaults, configurations, services, hosts):
validationItems = []
hive_env = properties
http://git-wip-us.apache.org/repos/asf/ambari/blob/16175189/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java
index 14b626b..3eea4d3 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java
@@ -19,26 +19,6 @@
package org.apache.ambari.server.upgrade;
-import javax.persistence.EntityManager;
-import junit.framework.Assert;
-import static org.easymock.EasyMock.anyObject;
-import static org.easymock.EasyMock.anyString;
-import static org.easymock.EasyMock.capture;
-import static org.easymock.EasyMock.createMockBuilder;
-import static org.easymock.EasyMock.createNiceMock;
-import static org.easymock.EasyMock.createStrictMock;
-import static org.easymock.EasyMock.eq;
-import static org.easymock.EasyMock.expect;
-import static org.easymock.EasyMock.expectLastCall;
-import static org.easymock.EasyMock.newCapture;
-import static org.easymock.EasyMock.replay;
-import static org.easymock.EasyMock.reset;
-import static org.easymock.EasyMock.verify;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-
import java.io.File;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
@@ -55,6 +35,8 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import javax.persistence.EntityManager;
+
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.actionmanager.ActionManager;
import org.apache.ambari.server.api.services.AmbariMetaInfo;
@@ -126,7 +108,32 @@ import com.google.inject.Injector;
import com.google.inject.Module;
import com.google.inject.Provider;
+import junit.framework.Assert;
+
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.anyString;
+import static org.easymock.EasyMock.capture;
+import static org.easymock.EasyMock.createMockBuilder;
+import static org.easymock.EasyMock.createNiceMock;
+import static org.easymock.EasyMock.createStrictMock;
+import static org.easymock.EasyMock.eq;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.expectLastCall;
+import static org.easymock.EasyMock.newCapture;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.reset;
+import static org.easymock.EasyMock.verify;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
public class UpgradeCatalog240Test {
+ private static final String CAPACITY_SCHEDULER_CONFIG_TYPE = "capacity-scheduler";
+ private static final String WEBHCAT_SITE_CONFIG_TYPE = "webhcat-site";
+ private static final String TEZ_SITE_CONFIG_TYPE = "tez-site";
+ private static final String MAPRED_SITE_CONFIG_TYPE = "mapred-site";
+ private static final String YARN_ENV_CONFIG_TYPE = "yarn-env";
private static Injector injector;
private Provider<EntityManager> entityManagerProvider = createStrictMock(Provider.class);
private EntityManager entityManager = createNiceMock(EntityManager.class);
@@ -541,6 +548,7 @@ public class UpgradeCatalog240Test {
Method updateHostRoleCommandTableDML = UpgradeCatalog240.class.getDeclaredMethod("updateHostRoleCommandTableDML");
Method updateKerberosEnv = UpgradeCatalog240.class.getDeclaredMethod("updateKerberosConfigs");
Method updateYarnEnv = UpgradeCatalog240.class.getDeclaredMethod("updateYarnEnv");
+ Method updateQueueNameConfigs = UpgradeCatalog240.class.getDeclaredMethod("updateQueueNameConfigs");
Method removeHiveOozieDBConnectionConfigs = UpgradeCatalog240.class.getDeclaredMethod("removeHiveOozieDBConnectionConfigs");
Method updateClustersAndHostsVersionStateTableDML = UpgradeCatalog240.class.getDeclaredMethod("updateClustersAndHostsVersionStateTableDML");
Method removeStandardDeviationAlerts = UpgradeCatalog240.class.getDeclaredMethod("removeStandardDeviationAlerts");
@@ -582,6 +590,7 @@ public class UpgradeCatalog240Test {
.addMockedMethod(updateHostRoleCommandTableDML)
.addMockedMethod(updateKerberosEnv)
.addMockedMethod(updateYarnEnv)
+ .addMockedMethod(updateQueueNameConfigs)
.addMockedMethod(removeHiveOozieDBConnectionConfigs)
.addMockedMethod(updateClustersAndHostsVersionStateTableDML)
.addMockedMethod(removeStandardDeviationAlerts)
@@ -618,6 +627,7 @@ public class UpgradeCatalog240Test {
upgradeCatalog240.updateHostRoleCommandTableDML();
upgradeCatalog240.updateKerberosConfigs();
upgradeCatalog240.updateYarnEnv();
+ upgradeCatalog240.updateQueueNameConfigs();
upgradeCatalog240.removeHiveOozieDBConnectionConfigs();
upgradeCatalog240.updateClustersAndHostsVersionStateTableDML();
upgradeCatalog240.removeStandardDeviationAlerts();
@@ -996,6 +1006,156 @@ public class UpgradeCatalog240Test {
assertTrue(Maps.difference(newPropertiesYarnEnv, updatedProperties).areEqual());
}
+ /**
+ * Test that queue names updated in mapred-site, webhcat-site, tez-site, yarn-env
+ * @throws Exception
+ */
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testQueueNameUpdateConfigs() throws Exception{
+ Map<String, String> oldPropertiesCapacityScheduler = new HashMap<String, String>() {
+ {
+ put("yarn.scheduler.capacity.root.queues", "default3,d3");
+ put("yarn.scheduler.capacity.root.d3.queues", "default4");
+ put("yarn.scheduler.capacity.root.default3.queues", "default5");
+ }
+ };
+ Map<String, String> oldPropertiesTezSite = new HashMap<String, String>() {
+ {
+ put("tez.queue.name", "default3");
+ }
+ };
+ Map<String, String> oldPropertiesWebhcatSite = new HashMap<String, String>() {
+ {
+ put("templeton.hadoop.queue.name", "default3");
+ }
+ };
+ Map<String, String> oldPropertiesMapredSite = new HashMap<String, String>() {
+ {
+ put("mapreduce.job.queuename", "default3");
+ }
+ };
+ Map<String, String> oldPropertiesYarnEnv = new HashMap<String, String>() {
+ {
+ put("service_check.queue.name", "default3");
+ }
+ };
+
+ Map<String, String> newPropertiesTezSite = new HashMap<String, String>() {
+ {
+ put("tez.queue.name", "default5");
+ }
+ };
+ Map<String, String> newPropertiesWebhcatSite = new HashMap<String, String>() {
+ {
+ put("templeton.hadoop.queue.name", "default5");
+ }
+ };
+ Map<String, String> newPropertiesMapredSite = new HashMap<String, String>() {
+ {
+ put("mapreduce.job.queuename", "default5");
+ }
+ };
+ Map<String, String> newPropertiesYarnEnv = new HashMap<String, String>() {
+ {
+ put("service_check.queue.name", "default5");
+ }
+ };
+
+ EasyMockSupport easyMockSupport = new EasyMockSupport();
+
+ Clusters clusters = easyMockSupport.createNiceMock(Clusters.class);
+ final Cluster cluster = easyMockSupport.createNiceMock(Cluster.class);
+ Config mockCapacityScheduler = easyMockSupport.createNiceMock(Config.class);
+ Config mockWebhcatSite = easyMockSupport.createNiceMock(Config.class);
+ Config mockTezSite = easyMockSupport.createNiceMock(Config.class);
+ Config mockMapredSite = easyMockSupport.createNiceMock(Config.class);
+ Config mockYarnEnv = easyMockSupport.createNiceMock(Config.class);
+
+ expect(clusters.getClusters()).andReturn(new HashMap<String, Cluster>() {{
+ put("normal", cluster);
+ }}).anyTimes();
+
+ expect(cluster.getServices()).andReturn(new HashMap<String, Service>() {
+ {
+ put("YARN", null);
+ put("HIVE", null);
+ put("MAPREDUCE", null);
+ put("TEZ", null);
+ }
+ }).atLeastOnce();
+
+ expect(cluster.getServiceByConfigType(WEBHCAT_SITE_CONFIG_TYPE)).andReturn("HIVE").atLeastOnce();
+ expect(cluster.getServiceByConfigType(TEZ_SITE_CONFIG_TYPE)).andReturn("TEZ").atLeastOnce();
+ expect(cluster.getServiceByConfigType(MAPRED_SITE_CONFIG_TYPE)).andReturn("MAPREDUCE").atLeastOnce();
+ expect(cluster.getServiceByConfigType(YARN_ENV_CONFIG_TYPE)).andReturn("YARN").atLeastOnce();
+
+ expect(cluster.getDesiredConfigByType(CAPACITY_SCHEDULER_CONFIG_TYPE)).andReturn(mockCapacityScheduler).atLeastOnce();
+ expect(cluster.getDesiredConfigByType(WEBHCAT_SITE_CONFIG_TYPE)).andReturn(mockWebhcatSite).atLeastOnce();
+ expect(cluster.getDesiredConfigByType(TEZ_SITE_CONFIG_TYPE)).andReturn(mockTezSite).atLeastOnce();
+ expect(cluster.getDesiredConfigByType(MAPRED_SITE_CONFIG_TYPE)).andReturn(mockMapredSite).atLeastOnce();
+ expect(cluster.getDesiredConfigByType(YARN_ENV_CONFIG_TYPE)).andReturn(mockYarnEnv).atLeastOnce();
+
+
+
+ expect(mockCapacityScheduler.getProperties()).andReturn(oldPropertiesCapacityScheduler).anyTimes();
+ expect(mockWebhcatSite.getProperties()).andReturn(oldPropertiesWebhcatSite).anyTimes();
+ expect(mockTezSite.getProperties()).andReturn(oldPropertiesTezSite).anyTimes();
+ expect(mockMapredSite.getProperties()).andReturn(oldPropertiesMapredSite).anyTimes();
+ expect(mockYarnEnv.getProperties()).andReturn(oldPropertiesYarnEnv).anyTimes();
+
+ Injector injector = easyMockSupport.createNiceMock(Injector.class);
+ expect(injector.getInstance(Gson.class)).andReturn(null).anyTimes();
+ expect(injector.getInstance(MaintenanceStateHelper.class)).andReturn(null).anyTimes();
+ expect(injector.getInstance(KerberosHelper.class)).andReturn(createNiceMock(KerberosHelper.class)).anyTimes();
+
+ replay(injector, clusters, mockCapacityScheduler, mockWebhcatSite, mockTezSite, mockMapredSite,
+ mockYarnEnv, cluster);
+
+ AmbariManagementControllerImpl controller = createMockBuilder(AmbariManagementControllerImpl.class)
+ .addMockedMethod("createConfiguration")
+ .addMockedMethod("getClusters", new Class[] { })
+ .addMockedMethod("createConfig")
+ .withConstructor(createNiceMock(ActionManager.class), clusters, injector)
+ .createNiceMock();
+
+ Injector injector2 = easyMockSupport.createNiceMock(Injector.class);
+ Capture<Map> propertiesCapacityScheduler = EasyMock.newCapture();
+ Capture<Map> propertiesCaptureYarnEnv = EasyMock.newCapture();
+ Capture<Map> propertiesCaptureWebhcatSite = EasyMock.newCapture();
+ Capture<Map> propertiesCaptureTezSite= EasyMock.newCapture();
+ Capture<Map> propertiesCaptureMapredSite = EasyMock.newCapture();
+
+ expect(injector2.getInstance(AmbariManagementController.class)).andReturn(controller).anyTimes();
+ expect(controller.getClusters()).andReturn(clusters).anyTimes();
+ expect(controller.createConfig(anyObject(Cluster.class), eq(CAPACITY_SCHEDULER_CONFIG_TYPE), capture(propertiesCapacityScheduler), anyString(),
+ anyObject(Map.class))).andReturn(createNiceMock(Config.class)).once();
+ expect(controller.createConfig(anyObject(Cluster.class), eq(YARN_ENV_CONFIG_TYPE), capture(propertiesCaptureYarnEnv), anyString(),
+ anyObject(Map.class))).andReturn(createNiceMock(Config.class)).once();
+ expect(controller.createConfig(anyObject(Cluster.class), eq(WEBHCAT_SITE_CONFIG_TYPE), capture(propertiesCaptureWebhcatSite), anyString(),
+ anyObject(Map.class))).andReturn(createNiceMock(Config.class)).once();
+ expect(controller.createConfig(anyObject(Cluster.class), eq(TEZ_SITE_CONFIG_TYPE), capture(propertiesCaptureTezSite), anyString(),
+ anyObject(Map.class))).andReturn(createNiceMock(Config.class)).once();
+ expect(controller.createConfig(anyObject(Cluster.class), eq(MAPRED_SITE_CONFIG_TYPE), capture(propertiesCaptureMapredSite), anyString(),
+ anyObject(Map.class))).andReturn(createNiceMock(Config.class)).once();
+
+ replay(controller, injector2);
+ new UpgradeCatalog240(injector2).updateQueueNameConfigs();
+ easyMockSupport.verifyAll();
+
+ Map<String, String> updatedPropertiesYarnEnv = propertiesCaptureYarnEnv.getValue();
+ assertTrue(Maps.difference(newPropertiesYarnEnv, updatedPropertiesYarnEnv).areEqual());
+
+ Map<String, String> updatedPropertiesWebhcatSite = propertiesCaptureWebhcatSite.getValue();
+ assertTrue(Maps.difference(newPropertiesWebhcatSite, updatedPropertiesWebhcatSite).areEqual());
+
+ Map<String, String> updatedPropertiesTezSite = propertiesCaptureTezSite.getValue();
+ assertTrue(Maps.difference(newPropertiesTezSite, updatedPropertiesTezSite).areEqual());
+
+ Map<String, String> updatedPropertiesMapredSite = propertiesCaptureMapredSite.getValue();
+ assertTrue(Maps.difference(newPropertiesMapredSite, updatedPropertiesMapredSite).areEqual());
+ }
+
@Test
public void testSparkConfigUpdate() throws Exception{
http://git-wip-us.apache.org/repos/asf/ambari/blob/16175189/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
index eee00de..6a8f3e2 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
@@ -218,57 +218,16 @@ class TestHDP206StackAdvisor(TestCase):
]
services = self.prepareServices(servicesInfo)
services["configurations"] = {"yarn-env":{"properties":{"service_check.queue.name": "default"}},
- "capacity-scheduler":{"properties":{"capacity-scheduler":
- "yarn.scheduler.capacity.ndfqueue.minimum-user-limit-percent=100\n" +
- "yarn.scheduler.capacity.maximum-am-resource-percent=0.2\n" +
- "yarn.scheduler.capacity.maximum-applications=10000\n" +
- "yarn.scheduler.capacity.node-locality-delay=40\n" +
- "yarn.scheduler.capacity.root.accessible-node-labels=*\n" +
- "yarn.scheduler.capacity.root.acl_administer_queue=*\n" +
- "yarn.scheduler.capacity.root.capacity=100\n" +
- "yarn.scheduler.capacity.root.ndfqueue.acl_administer_jobs=*\n" +
- "yarn.scheduler.capacity.root.ndfqueue.acl_submit_applications=*\n" +
- "yarn.scheduler.capacity.root.ndfqueue.capacity=100\n" +
- "yarn.scheduler.capacity.root.ndfqueue.maximum-capacity=100\n" +
- "yarn.scheduler.capacity.root.ndfqueue.state=RUNNING\n" +
- "yarn.scheduler.capacity.root.ndfqueue.user-limit-factor=1\n" +
- "yarn.scheduler.capacity.root.queues=ndfqueue\n"}}}
+ "capacity-scheduler":{"properties":{"capacity-scheduler": "yarn.scheduler.capacity.root.queues=ndfqueue\n"}}}
hosts = self.prepareHosts([])
result = self.stackAdvisor.validateConfigurations(services, hosts)
-
expectedItems = [
- {'message': 'service_check.queue.name is not exist, or not corresponds to existing leaf queue', 'level': 'ERROR'}
+ {'message': 'Queue is not exist, or not corresponds to existing YARN leaf queue', 'level': 'ERROR'}
]
self.assertValidationResult(expectedItems, result)
-
- def test_validationYARNServicecheckQueueNameDefault(self):
- servicesInfo = [
- {
- "name": "YARN",
- "components": []
- }
- ]
- services = self.prepareServices(servicesInfo)
- services["configurations"] = {"yarn-env":{"properties":{"service_check.queue.name": "default"}},
- "capacity-scheduler":{"properties":{"capacity-scheduler":
- "yarn.scheduler.capacity.default.minimum-user-limit-percent=100\n" +
- "yarn.scheduler.capacity.maximum-am-resource-percent=0.2\n" +
- "yarn.scheduler.capacity.maximum-applications=10000\n" +
- "yarn.scheduler.capacity.node-locality-delay=40\n" +
- "yarn.scheduler.capacity.root.accessible-node-labels=*\n" +
- "yarn.scheduler.capacity.root.acl_administer_queue=*\n" +
- "yarn.scheduler.capacity.root.capacity=100\n" +
- "yarn.scheduler.capacity.root.default.acl_administer_jobs=*\n" +
- "yarn.scheduler.capacity.root.default.acl_submit_applications=*\n" +
- "yarn.scheduler.capacity.root.default.capacity=100\n" +
- "yarn.scheduler.capacity.root.default.maximum-capacity=100\n" +
- "yarn.scheduler.capacity.root.default.state=RUNNING\n" +
- "yarn.scheduler.capacity.root.default.user-limit-factor=1\n" +
- "yarn.scheduler.capacity.root.queues=default\n"}}}
- hosts = self.prepareHosts([])
+ services["configurations"]["yarn-env"]["properties"]["service_check.queue.name"] = "ndfqueue"
+ expectedItems = []
result = self.stackAdvisor.validateConfigurations(services, hosts)
-
- expectedItems = [ ]
self.assertValidationResult(expectedItems, result)
def test_validationMinMax(self):
@@ -581,7 +540,8 @@ class TestHDP206StackAdvisor(TestCase):
expected = {
"yarn-env": {
"properties": {
- "min_user_id": "500"
+ "min_user_id": "500",
+ 'service_check.queue.name': 'default'
}
},
"yarn-site": {
@@ -607,6 +567,7 @@ class TestHDP206StackAdvisor(TestCase):
expected = {
"mapred-site": {
"properties": {
+ 'mapreduce.job.queuename': 'default',
"yarn.app.mapreduce.am.resource.mb": "123",
"yarn.app.mapreduce.am.command-opts": "-Xmx99m",
"mapreduce.map.memory.mb": "567",
http://git-wip-us.apache.org/repos/asf/ambari/blob/16175189/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
index a7e4072..08b9554 100644
--- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
@@ -95,6 +95,7 @@ class TestHDP22StackAdvisor(TestCase):
"tez-site": {
"properties": {
# tez.am.resource.memory.mb must be <= yarn.scheduler.maximum-allocation-mb
+ 'tez.queue.name': 'default',
"tez.am.resource.memory.mb": "2048",
"tez.task.resource.memory.mb": "768",
"tez.runtime.io.sort.mb": "307",
@@ -188,6 +189,7 @@ class TestHDP22StackAdvisor(TestCase):
"tez-site": {
"properties": {
# tez.am.resource.memory.mb must be <= yarn.scheduler.maximum-allocation-mb
+ 'tez.queue.name': 'default',
"tez.am.resource.memory.mb": "2048",
"tez.task.resource.memory.mb": "768",
"tez.runtime.io.sort.mb": "307",
@@ -272,6 +274,7 @@ class TestHDP22StackAdvisor(TestCase):
"tez-site": {
"properties": {
# tez.am.resource.memory.mb must be <= yarn.scheduler.maximum-allocation-mb
+ 'tez.queue.name': 'default',
"tez.am.resource.memory.mb": "2048",
"tez.task.resource.memory.mb": "760",
"tez.runtime.io.sort.mb": "304",
@@ -865,7 +868,8 @@ class TestHDP22StackAdvisor(TestCase):
expected = {
"yarn-env": {
"properties": {
- "min_user_id": "500"
+ "min_user_id": "500",
+ 'service_check.queue.name': 'default'
}
},
"yarn-site": {
@@ -909,7 +913,8 @@ class TestHDP22StackAdvisor(TestCase):
expected = {
"yarn-env": {
"properties": {
- "min_user_id": "500"
+ "min_user_id": "500",
+ 'service_check.queue.name': 'default'
}
},
"yarn-site": {
@@ -1226,6 +1231,11 @@ class TestHDP22StackAdvisor(TestCase):
'hive.security.authenticator.manager': {'delete': 'true'},
'hive.conf.restricted.list': {'delete': 'true'}
}
+ },
+ 'webhcat-site': {
+ 'properties': {
+ 'templeton.hadoop.queue.name': 'queue1'
+ }
}
}
@@ -1606,11 +1616,13 @@ class TestHDP22StackAdvisor(TestCase):
},
"yarn-env": {
"properties": {
- "min_user_id": "500"
+ "min_user_id": "500",
+ 'service_check.queue.name': 'default'
}
},
"mapred-site": {
"properties": {
+ 'mapreduce.job.queuename': 'default',
"mapreduce.map.memory.mb": "1536",
"mapreduce.reduce.memory.mb": "1536",
"yarn.app.mapreduce.am.command-opts": "-Xmx80m -Dhdp.version=${hdp.version}",
@@ -1867,11 +1879,13 @@ class TestHDP22StackAdvisor(TestCase):
expected = {
"yarn-env": {
"properties": {
- "min_user_id": "500"
+ "min_user_id": "500",
+ 'service_check.queue.name': 'default'
}
},
"mapred-site": {
"properties": {
+ 'mapreduce.job.queuename': 'default',
"mapreduce.map.memory.mb": "100",
"mapreduce.reduce.memory.mb": "200",
"yarn.app.mapreduce.am.command-opts": "-Xmx80m -Dhdp.version=${hdp.version}",
@@ -2080,11 +2094,13 @@ class TestHDP22StackAdvisor(TestCase):
expected = {
"yarn-env": {
"properties": {
- "min_user_id": "500"
+ "min_user_id": "500",
+ 'service_check.queue.name': 'default'
}
},
"mapred-site": {
"properties": {
+ 'mapreduce.job.queuename': 'default',
"mapreduce.map.memory.mb": "700",
"mapreduce.reduce.memory.mb": "1280",
"yarn.app.mapreduce.am.command-opts": "-Xmx560m -Dhdp.version=${hdp.version}",
@@ -3264,7 +3280,12 @@ class TestHDP22StackAdvisor(TestCase):
'tez.tez-ui.history-url.base' : 'http://host:8080/#/main/views/TEZ/0.7.0.2.3.0.0-2155/TEZ_CLUSTER_INSTANCE'}
- res_expected = [{'config-name': 'tez.tez-ui.history-url.base',
+ res_expected = [{'config-name': 'tez.queue.name',
+ 'config-type': 'tez-site',
+ 'level': 'ERROR',
+ 'message': 'Value should be set',
+ 'type': 'configuration'},
+ {'config-name': 'tez.tez-ui.history-url.base',
'config-type': 'tez-site',
'level': 'WARN',
'message': "It is recommended to set value https://host:8443/#/main/views/TEZ/0.7.0.2.3.0.0-2155/TEZ_CLUSTER_INSTANCE for property tez.tez-ui.history-url.base",
@@ -3318,15 +3339,18 @@ class TestHDP22StackAdvisor(TestCase):
def test_validateYARNConfigurationsEnv(self):
configurations = {}
+ services = {}
+ services['configurations'] = configurations
# 1) ok: No yarn_cgroups_enabled
recommendedDefaults = {'namenode_heapsize': '1024',
'namenode_opt_newsize' : '256',
'namenode_opt_maxnewsize' : '256'}
properties = {}
+ properties['service_check.queue.name'] = 'default'
res_expected = []
- res = self.stackAdvisor.validateYARNEnvConfigurations(properties, recommendedDefaults, configurations, '', '')
+ res = self.stackAdvisor.validateYARNEnvConfigurations(properties, recommendedDefaults, configurations, services, '')
self.assertEquals(res, res_expected)
# 2) ok: yarn_cgroups_enabled=false, but security enabled
@@ -3340,13 +3364,13 @@ class TestHDP22StackAdvisor(TestCase):
}
}
res_expected = []
- res = self.stackAdvisor.validateYARNEnvConfigurations(properties, recommendedDefaults, configurations, '', '')
+ res = self.stackAdvisor.validateYARNEnvConfigurations(properties, recommendedDefaults, configurations, services, '')
self.assertEquals(res, res_expected)
# 3) ok: yarn_cgroups_enabled=true, but security enabled
properties['yarn_cgroups_enabled'] = 'true'
res_expected = []
- res = self.stackAdvisor.validateYARNEnvConfigurations(properties, recommendedDefaults, configurations, '', '')
+ res = self.stackAdvisor.validateYARNEnvConfigurations(properties, recommendedDefaults, configurations, services, '')
self.assertEquals(res, res_expected)
# 4) fail: yarn_cgroups_enabled=true, but security disabled
@@ -3356,7 +3380,7 @@ class TestHDP22StackAdvisor(TestCase):
'type': 'configuration',
'config-name': 'yarn_cgroups_enabled',
'level': 'WARN'}]
- res = self.stackAdvisor.validateYARNEnvConfigurations(properties, recommendedDefaults, configurations, '', '')
+ res = self.stackAdvisor.validateYARNEnvConfigurations(properties, recommendedDefaults, configurations, services, '')
self.assertEquals(res, res_expected)
def test_validateMR2XmxOptsEnv(self):
@@ -3390,6 +3414,11 @@ class TestHDP22StackAdvisor(TestCase):
'type': 'configuration',
'config-name': 'yarn.app.mapreduce.am.command-opts',
'level': 'WARN'},
+ {'config-name': 'mapreduce.job.queuename',
+ 'config-type': 'mapred-site',
+ 'level': 'ERROR',
+ 'message': 'Value should be set',
+ 'type': 'configuration'},
{'config-type': 'mapred-site',
'message': 'yarn.app.mapreduce.am.command-opts Xmx should be less than yarn.app.mapreduce.am.resource.mb (410)',
'type': 'configuration',
@@ -3571,7 +3600,8 @@ class TestHDP22StackAdvisor(TestCase):
expected = {
"yarn-env": {
"properties": {
- "min_user_id": "500"
+ "min_user_id": "500",
+ "service_check.queue.name": "default"
}
},
"yarn-site": {
@@ -3626,7 +3656,8 @@ class TestHDP22StackAdvisor(TestCase):
expected = {
"yarn-env": {
"properties": {
- "min_user_id": "500"
+ "min_user_id": "500",
+ 'service_check.queue.name': 'default'
}
},
"yarn-site": {
http://git-wip-us.apache.org/repos/asf/ambari/blob/16175189/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
index 9d830da..62d926e 100644
--- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
@@ -1342,6 +1342,11 @@ class TestHDP23StackAdvisor(TestCase):
'hive.security.authorization.manager': {'delete': 'true'},
'hive.security.authenticator.manager': {'delete': 'true'}
}
+ },
+ 'webhcat-site': {
+ 'properties': {
+ 'templeton.hadoop.queue.name': 'queue1'
+ }
}
}
services = {
@@ -1603,6 +1608,11 @@ class TestHDP23StackAdvisor(TestCase):
'hive.security.authorization.manager': {'delete': 'true'},
'hive.security.authenticator.manager': {'delete': 'true'}
}
+ },
+ 'webhcat-site': {
+ 'properties': {
+ 'templeton.hadoop.queue.name': 'queue1'
+ }
}
}
services = {
@@ -1821,7 +1831,8 @@ class TestHDP23StackAdvisor(TestCase):
"tez.runtime.io.sort.mb": "307",
"tez.session.am.dag.submit.timeout.secs": "600",
"tez.runtime.unordered.output.buffer.size-mb": "57",
- "tez.am.resource.memory.mb": "4000"
+ "tez.am.resource.memory.mb": "4000",
+ "tez.queue.name": "queue1",
}
},
"yarn-site": {
http://git-wip-us.apache.org/repos/asf/ambari/blob/16175189/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
index 1bc53ea..c9c21ae 100644
--- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
@@ -7130,27 +7130,17 @@ class TestHDP25StackAdvisor(TestCase):
]
services = self.prepareServices(servicesInfo)
services["configurations"] = {"yarn-env":{"properties":{"service_check.queue.name": "default"}},
- "capacity-scheduler":{"properties":{"capacity-scheduler":
- "yarn.scheduler.capacity.ndfqueue.minimum-user-limit-percent=100\n" +
- "yarn.scheduler.capacity.maximum-am-resource-percent=0.2\n" +
- "yarn.scheduler.capacity.maximum-applications=10000\n" +
- "yarn.scheduler.capacity.node-locality-delay=40\n" +
- "yarn.scheduler.capacity.root.accessible-node-labels=*\n" +
- "yarn.scheduler.capacity.root.acl_administer_queue=*\n" +
- "yarn.scheduler.capacity.root.capacity=100\n" +
- "yarn.scheduler.capacity.root.ndfqueue.acl_administer_jobs=*\n" +
- "yarn.scheduler.capacity.root.ndfqueue.acl_submit_applications=*\n" +
- "yarn.scheduler.capacity.root.ndfqueue.capacity=100\n" +
- "yarn.scheduler.capacity.root.ndfqueue.maximum-capacity=100\n" +
- "yarn.scheduler.capacity.root.ndfqueue.state=RUNNING\n" +
- "yarn.scheduler.capacity.root.ndfqueue.user-limit-factor=1\n" +
- "yarn.scheduler.capacity.root.queues=ndfqueue\n"}}}
+ "capacity-scheduler":{"properties":{"capacity-scheduler": "yarn.scheduler.capacity.root.queues=ndfqueue\n"}}}
hosts = self.prepareHosts([])
result = self.stackAdvisor.validateConfigurations(services, hosts)
expectedItems = [
- {'message': 'service_check.queue.name is not exist, or not corresponds to existing leaf queue', 'level': 'ERROR'}
+ {'message': 'Queue is not exist, or not corresponds to existing YARN leaf queue', 'level': 'ERROR'}
]
self.assertValidationResult(expectedItems, result)
+ services["configurations"]["yarn-env"]["properties"]["service_check.queue.name"] = "ndfqueue"
+ expectedItems = []
+ result = self.stackAdvisor.validateConfigurations(services, hosts)
+ self.assertValidationResult(expectedItems, result)
def assertValidationResult(self, expectedItems, result):
actualItems = []