You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by bi...@apache.org on 2013/11/15 20:48:12 UTC
svn commit: r1542380 - in
/hadoop/common/branches/branch-2/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/
hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/
hadoop-yarn/hadoop-yarn-...
Author: bikas
Date: Fri Nov 15 19:48:12 2013
New Revision: 1542380
URL: http://svn.apache.org/r1542380
Log:
Merge r1542367 from trunk to branch-2 for YARN-1411. HA config shouldn't affect NodeManager RPC addresses (Karthik Kambatla via bikas)
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt?rev=1542380&r1=1542379&r2=1542380&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt Fri Nov 15 19:48:12 2013
@@ -118,6 +118,9 @@ Release 2.3.0 - UNRELEASED
YARN-1401. With zero sleep-delay-before-sigkill.ms, no signal is ever sent
(Gera Shegalov via Sandy Ryza)
+ YARN-1411. HA config shouldn't affect NodeManager RPC addresses (Karthik
+ Kambatla via bikas)
+
Release 2.2.1 - UNRELEASED
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java?rev=1542380&r1=1542379&r2=1542380&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java Fri Nov 15 19:48:12 2013
@@ -23,29 +23,14 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
-import java.util.Arrays;
import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
@InterfaceAudience.Private
public class HAUtil {
private static Log LOG = LogFactory.getLog(HAUtil.class);
- public static final List<String> RPC_ADDRESS_CONF_KEYS =
- Collections.unmodifiableList(Arrays.asList(
- YarnConfiguration.RM_ADDRESS,
- YarnConfiguration.RM_SCHEDULER_ADDRESS,
- YarnConfiguration.RM_ADMIN_ADDRESS,
- YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
- YarnConfiguration.RM_WEBAPP_ADDRESS,
- YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS,
- // TODO Remove after YARN-1318
- YarnConfiguration.RM_HA_ADMIN_ADDRESS));
-
public static final String BAD_CONFIG_MESSAGE_PREFIX =
"Invalid configuration! ";
@@ -139,7 +124,7 @@ public class HAUtil {
}
public static void verifyAndSetAllRpcAddresses(Configuration conf) {
- for (String confKey : RPC_ADDRESS_CONF_KEYS) {
+ for (String confKey : YarnConfiguration.RM_RPC_ADDRESS_CONF_KEYS) {
verifyAndSetConfValue(confKey, conf);
}
}
@@ -188,9 +173,12 @@ public class HAUtil {
ids.toString() + ")";
}
- private static String getConfKeyForRMInstance(String prefix,
- Configuration conf) {
- return addSuffix(prefix, getRMHAId(conf));
+ @InterfaceAudience.Private
+ @VisibleForTesting
+ static String getConfKeyForRMInstance(String prefix, Configuration conf) {
+ return YarnConfiguration.RM_RPC_ADDRESS_CONF_KEYS.contains(prefix)
+ ? addSuffix(prefix, getRMHAId(conf))
+ : prefix;
}
public static String getConfValueForRMInstance(String prefix,
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1542380&r1=1542379&r2=1542380&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java Fri Nov 15 19:48:12 2013
@@ -20,6 +20,8 @@ package org.apache.hadoop.yarn.conf;
import java.net.InetSocketAddress;
import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
@@ -295,6 +297,17 @@ public class YarnConfiguration extends C
public static final int DEFAULT_RM_HA_ADMIN_CLIENT_THREAD_COUNT = 1;
// end @Private
+ public static final List<String> RM_RPC_ADDRESS_CONF_KEYS =
+ Collections.unmodifiableList(Arrays.asList(
+ RM_ADDRESS,
+ RM_SCHEDULER_ADDRESS,
+ RM_ADMIN_ADDRESS,
+ RM_RESOURCE_TRACKER_ADDRESS,
+ RM_WEBAPP_ADDRESS,
+ RM_WEBAPP_HTTPS_ADDRESS,
+ // TODO Remove after YARN-1318
+ RM_HA_ADMIN_ADDRESS));
+
////////////////////////////////
// RM state store configs
////////////////////////////////
@@ -924,7 +937,7 @@ public class YarnConfiguration extends C
public InetSocketAddress getSocketAddr(
String name, String defaultAddress, int defaultPort) {
String address;
- if (HAUtil.isHAEnabled(this)) {
+ if (HAUtil.isHAEnabled(this) && RM_RPC_ADDRESS_CONF_KEYS.contains(name)) {
address = HAUtil.getConfValueForRMInstance(name, defaultAddress, this);
} else {
address = get(name, defaultAddress);
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java?rev=1542380&r1=1542379&r2=1542380&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java Fri Nov 15 19:48:12 2013
@@ -28,7 +28,9 @@ import org.junit.Test;
import java.util.Collection;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
public class TestHAUtil {
@@ -51,7 +53,7 @@ public class TestHAUtil {
conf.set(YarnConfiguration.RM_HA_IDS, RM_NODE_IDS_UNTRIMMED);
conf.set(YarnConfiguration.RM_HA_ID, RM1_NODE_ID_UNTRIMMED);
- for (String confKey : HAUtil.RPC_ADDRESS_CONF_KEYS) {
+ for (String confKey : YarnConfiguration.RM_RPC_ADDRESS_CONF_KEYS) {
// configuration key itself cannot contains space/tab/return chars.
conf.set(HAUtil.addSuffix(confKey, RM1_NODE_ID), RM1_ADDRESS_UNTRIMMED);
conf.set(HAUtil.addSuffix(confKey, RM2_NODE_ID), RM2_ADDRESS);
@@ -92,7 +94,7 @@ public class TestHAUtil {
StringUtils.getStringCollection(RM_NODE_IDS), HAUtil.getRMHAIds(conf));
assertEquals("Should be saved as Trimmed string",
RM1_NODE_ID, HAUtil.getRMHAId(conf));
- for (String confKey : HAUtil.RPC_ADDRESS_CONF_KEYS) {
+ for (String confKey : YarnConfiguration.RM_RPC_ADDRESS_CONF_KEYS) {
assertEquals("RPC address not set for " + confKey,
RM1_ADDRESS, conf.get(confKey));
}
@@ -111,7 +113,7 @@ public class TestHAUtil {
conf.clear();
conf.set(YarnConfiguration.RM_HA_ID, RM_INVALID_NODE_ID);
conf.set(YarnConfiguration.RM_HA_IDS, RM_INVALID_NODE_ID);
- for (String confKey : HAUtil.RPC_ADDRESS_CONF_KEYS) {
+ for (String confKey : YarnConfiguration.RM_RPC_ADDRESS_CONF_KEYS) {
// simulate xml with invalid node id
conf.set(confKey + RM_INVALID_NODE_ID, RM_INVALID_NODE_ID);
}
@@ -126,7 +128,7 @@ public class TestHAUtil {
}
conf.clear();
- // simulate the case HAUtil.RPC_ADDRESS_CONF_KEYS are not set
+ // simulate the case HAUtil.RM_RPC_ADDRESS_CONF_KEYS are not set
conf.set(YarnConfiguration.RM_HA_ID, RM1_NODE_ID);
conf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID);
try {
@@ -145,7 +147,7 @@ public class TestHAUtil {
conf.clear();
conf.set(YarnConfiguration.RM_HA_IDS, RM2_NODE_ID + "," + RM3_NODE_ID);
conf.set(YarnConfiguration.RM_HA_ID, RM1_NODE_ID_UNTRIMMED);
- for (String confKey : HAUtil.RPC_ADDRESS_CONF_KEYS) {
+ for (String confKey : YarnConfiguration.RM_RPC_ADDRESS_CONF_KEYS) {
conf.set(HAUtil.addSuffix(confKey, RM1_NODE_ID), RM1_ADDRESS_UNTRIMMED);
conf.set(HAUtil.addSuffix(confKey, RM2_NODE_ID), RM2_ADDRESS);
}
@@ -158,4 +160,14 @@ public class TestHAUtil {
e.getMessage());
}
}
+
+ @Test
+ public void testGetConfKeyForRMInstance() {
+ assertTrue("RM instance id is not suffixed",
+ HAUtil.getConfKeyForRMInstance(YarnConfiguration.RM_ADDRESS, conf)
+ .contains(HAUtil.getRMHAId(conf)));
+ assertFalse("RM instance id is suffixed",
+ HAUtil.getConfKeyForRMInstance(YarnConfiguration.NM_ADDRESS, conf)
+ .contains(HAUtil.getRMHAId(conf)));
+ }
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java?rev=1542380&r1=1542379&r2=1542380&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java Fri Nov 15 19:48:12 2013
@@ -23,6 +23,12 @@ import junit.framework.Assert;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import org.junit.Test;
+import java.net.InetSocketAddress;
+import java.net.SocketAddress;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
public class TestYarnConfiguration {
@Test
@@ -52,4 +58,21 @@ public class TestYarnConfiguration {
"http://rmtesting:24543", rmWebUrl);
}
+ @Test
+ public void testGetSocketAddressForNMWithHA() {
+ YarnConfiguration conf = new YarnConfiguration();
+
+ // Set NM address
+ conf.set(YarnConfiguration.NM_ADDRESS, "0.0.0.0:1234");
+
+ // Set HA
+ conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
+ conf.set(YarnConfiguration.RM_HA_ID, "rm1");
+ assertTrue(HAUtil.isHAEnabled(conf));
+
+ InetSocketAddress addr = conf.getSocketAddr(YarnConfiguration.NM_ADDRESS,
+ YarnConfiguration.DEFAULT_NM_ADDRESS,
+ YarnConfiguration.DEFAULT_NM_PORT);
+ assertEquals(1234, addr.getPort());
+ }
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java?rev=1542380&r1=1542379&r2=1542380&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java Fri Nov 15 19:48:12 2013
@@ -51,7 +51,7 @@ public class TestRMHA {
Configuration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
conf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID);
- for (String confKey : HAUtil.RPC_ADDRESS_CONF_KEYS) {
+ for (String confKey : YarnConfiguration.RM_RPC_ADDRESS_CONF_KEYS) {
conf.set(HAUtil.addSuffix(confKey, RM1_NODE_ID), RM1_ADDRESS);
}
conf.set(YarnConfiguration.RM_HA_ID, RM1_NODE_ID);
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java?rev=1542380&r1=1542379&r2=1542380&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java Fri Nov 15 19:48:12 2013
@@ -104,7 +104,7 @@ public class TestZKRMStateStore extends
conf.set(YarnConfiguration.RM_STORE, ZKRMStateStore.class.getName());
conf.set(YarnConfiguration.ZK_RM_STATE_STORE_ADDRESS, hostPort);
conf.set(YarnConfiguration.RM_HA_ID, rmId);
- for (String rpcAddress : HAUtil.RPC_ADDRESS_CONF_KEYS) {
+ for (String rpcAddress : YarnConfiguration.RM_RPC_ADDRESS_CONF_KEYS) {
conf.set(HAUtil.addSuffix(rpcAddress, rmId), "localhost:0");
}
conf.set(YarnConfiguration.RM_HA_ADMIN_ADDRESS, "localhost:" + adminPort);