You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by vi...@apache.org on 2013/12/14 00:55:31 UTC
svn commit: r1550855 - in
/hadoop/common/branches/branch-2/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/
hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/
hadoop-yarn/hadoop-yarn-...
Author: vinodkv
Date: Fri Dec 13 23:55:31 2013
New Revision: 1550855
URL: http://svn.apache.org/r1550855
Log:
YARN-1485. Modified RM HA configuration validation to also ensure that service-address configuration are configured for every RM. Contributed by Xuan Gong.
svn merge --ignore-ancestry -c 1550854 ../../trunk/
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt?rev=1550855&r1=1550854&r2=1550855&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt Fri Dec 13 23:55:31 2013
@@ -150,6 +150,10 @@ Release 2.4.0 - UNRELEASED
YARN-1311. Fixed app specific scheduler-events' names to be app-attempt
based. (vinodkv via jianhe)
+ YARN-1485. Modified RM HA configuration validation to also ensure that
+ service-address configuration are configured for every RM. (Xuan Gong via
+ vinodkv)
+
OPTIMIZATIONS
BUG FIXES
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java?rev=1550855&r1=1550854&r2=1550855&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java Fri Dec 13 23:55:31 2013
@@ -58,13 +58,17 @@ public class HAUtil {
*/
public static void verifyAndSetConfiguration(Configuration conf)
throws YarnRuntimeException {
- verifyAndSetRMHAIds(conf);
- verifyAndSetRMHAId(conf);
+ verifyAndSetRMHAIdsList(conf);
+ verifyAndSetCurrentRMHAId(conf);
verifyAndSetAllServiceAddresses(conf);
}
-
- private static void verifyAndSetRMHAIds(Configuration conf) {
+ /**
+ * Verify configuration that there are at least two RM-ids
+ * and RPC addresses are specified for each RM-id.
+ * Then set the RM-ids.
+ */
+ private static void verifyAndSetRMHAIdsList(Configuration conf) {
Collection<String> ids =
conf.getTrimmedStringCollection(YarnConfiguration.RM_HA_IDS);
if (ids.size() < 2) {
@@ -76,6 +80,24 @@ public class HAUtil {
StringBuilder setValue = new StringBuilder();
for (String id: ids) {
+ // verify the RM service addresses configurations for every RMIds
+ for (String prefix : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) {
+ String confKey = null;
+ try {
+ confKey = addSuffix(prefix, id);
+ if (conf.getTrimmed(confKey) == null) {
+ throwBadConfigurationException(getNeedToSetValueMessage(confKey));
+ }
+ } catch (IllegalArgumentException iae) {
+ String errmsg = iae.getMessage();
+ if (confKey == null) {
+ // Error at addSuffix
+ errmsg = getInvalidValueMessage(YarnConfiguration.RM_HA_ID,
+ getRMHAId(conf));
+ }
+ throwBadConfigurationException(errmsg);
+ }
+ }
setValue.append(id);
setValue.append(",");
}
@@ -83,7 +105,7 @@ public class HAUtil {
setValue.substring(0, setValue.length() - 1));
}
- private static void verifyAndSetRMHAId(Configuration conf) {
+ private static void verifyAndSetCurrentRMHAId(Configuration conf) {
String rmId = conf.getTrimmed(YarnConfiguration.RM_HA_ID);
if (rmId == null) {
throwBadConfigurationException(
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java?rev=1550855&r1=1550854&r2=1550855&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java Fri Dec 13 23:55:31 2013
@@ -39,6 +39,7 @@ public class TestHAUtil {
private static final String RM1_ADDRESS_UNTRIMMED = " \t\t\n 1.2.3.4:8021 \n\t ";
private static final String RM1_ADDRESS = RM1_ADDRESS_UNTRIMMED.trim();
private static final String RM2_ADDRESS = "localhost:8022";
+ private static final String RM3_ADDRESS = "localhost:8033";
private static final String RM1_NODE_ID_UNTRIMMED = "rm1 ";
private static final String RM1_NODE_ID = RM1_NODE_ID_UNTRIMMED.trim();
private static final String RM2_NODE_ID = "rm2";
@@ -113,8 +114,13 @@ public class TestHAUtil {
}
conf.clear();
- conf.set(YarnConfiguration.RM_HA_IDS, RM_INVALID_NODE_ID + ","
- + RM1_NODE_ID);
+ // simulate the case YarnConfiguration.RM_HA_ID is not set
+ conf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + ","
+ + RM2_NODE_ID);
+ for (String confKey : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) {
+ conf.set(HAUtil.addSuffix(confKey, RM1_NODE_ID), RM1_ADDRESS);
+ conf.set(HAUtil.addSuffix(confKey, RM2_NODE_ID), RM2_ADDRESS);
+ }
try {
HAUtil.verifyAndSetConfiguration(conf);
} catch (YarnRuntimeException e) {
@@ -165,6 +171,7 @@ public class TestHAUtil {
for (String confKey : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) {
conf.set(HAUtil.addSuffix(confKey, RM1_NODE_ID), RM1_ADDRESS_UNTRIMMED);
conf.set(HAUtil.addSuffix(confKey, RM2_NODE_ID), RM2_ADDRESS);
+ conf.set(HAUtil.addSuffix(confKey, RM3_NODE_ID), RM3_ADDRESS);
}
try {
HAUtil.verifyAndSetConfiguration(conf);
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java?rev=1550855&r1=1550854&r2=1550855&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java Fri Dec 13 23:55:31 2013
@@ -143,7 +143,9 @@ public class TestZKRMStateStore extends
conf.set(YarnConfiguration.ZK_RM_STATE_STORE_ADDRESS, hostPort);
conf.set(YarnConfiguration.RM_HA_ID, rmId);
for (String rpcAddress : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) {
- conf.set(HAUtil.addSuffix(rpcAddress, rmId), "localhost:0");
+ for (String id : HAUtil.getRMHAIds(conf)) {
+ conf.set(HAUtil.addSuffix(rpcAddress, id), "localhost:0");
+ }
}
conf.set(HAUtil.addSuffix(YarnConfiguration.RM_ADMIN_ADDRESS, rmId),
"localhost:" + adminPort);
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java?rev=1550855&r1=1550854&r2=1550855&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java Fri Dec 13 23:55:31 2013
@@ -296,7 +296,9 @@ public class MiniYARNCluster extends Com
String hostname = MiniYARNCluster.getHostname();
conf.set(YarnConfiguration.RM_HA_ID, rmId);
for (String confKey : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) {
- conf.set(HAUtil.addSuffix(confKey, rmId), hostname + ":0");
+ for (String id : HAUtil.getRMHAIds(conf)) {
+ conf.set(HAUtil.addSuffix(confKey, id), hostname + ":0");
+ }
}
}