You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zj...@apache.org on 2015/06/18 20:25:04 UTC
[26/50] [abbrv] hadoop git commit: YARN-3714. AM proxy filter can not
get RM webapp address from yarn.resourcemanager.hostname.rm-id. Contributed
by Masatake Iwasaki
YARN-3714. AM proxy filter can not get RM webapp address from
yarn.resourcemanager.hostname.rm-id. Contributed by Masatake Iwasaki
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/76dc2f53
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/76dc2f53
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/76dc2f53
Branch: refs/heads/YARN-2928
Commit: 76dc2f535343a806db5cc86f2d270c7d00412bee
Parents: 816ab87
Author: Xuan <xg...@apache.org>
Authored: Tue Jun 16 14:05:09 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Thu Jun 18 11:18:58 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ++
.../org/apache/hadoop/yarn/util/RMHAUtils.java | 33 ++++++++------
.../amfilter/TestAmFilterInitializer.java | 47 ++++++++++++++++++++
3 files changed, 70 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/76dc2f53/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 92060ae..3679bf8 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -627,6 +627,9 @@ Release 2.8.0 - UNRELEASED
YARN-3794. TestRMEmbeddedElector fails because of ambiguous LOG reference.
(Chengbing Liu via devaraj)
+ YARN-3714. AM proxy filter can not get RM webapp address from
+ yarn.resourcemanager.hostname.rm-id. (Masatake Iwasaki via xgong)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/76dc2f53/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/RMHAUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/RMHAUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/RMHAUtils.java
index a7e1ce9..2e996e9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/RMHAUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/RMHAUtils.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.ha.HAServiceProtocol;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.ha.HAServiceTarget;
import org.apache.hadoop.yarn.client.RMHAServiceTarget;
+import org.apache.hadoop.yarn.conf.HAUtil;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@Private
@@ -71,25 +72,31 @@ public class RMHAUtils {
public static List<String> getRMHAWebappAddresses(
final YarnConfiguration conf) {
+ String prefix;
+ String defaultPort;
+ if (YarnConfiguration.useHttps(conf)) {
+ prefix = YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS;
+ defaultPort = ":" + YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_PORT;
+ } else {
+ prefix =YarnConfiguration.RM_WEBAPP_ADDRESS;
+ defaultPort = ":" + YarnConfiguration.DEFAULT_RM_WEBAPP_PORT;
+ }
Collection<String> rmIds =
conf.getStringCollection(YarnConfiguration.RM_HA_IDS);
List<String> addrs = new ArrayList<String>();
- if (YarnConfiguration.useHttps(conf)) {
- for (String id : rmIds) {
- String addr = conf.get(
- YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS + "." + id);
- if (addr != null) {
- addrs.add(addr);
+ for (String id : rmIds) {
+ String addr = conf.get(HAUtil.addSuffix(prefix, id));
+ if (addr == null) {
+ String hostname =
+ conf.get(HAUtil.addSuffix(YarnConfiguration.RM_HOSTNAME, id));
+ if (hostname != null) {
+ addr = hostname + defaultPort;
}
}
- } else {
- for (String id : rmIds) {
- String addr = conf.get(YarnConfiguration.RM_WEBAPP_ADDRESS + "." + id);
- if (addr != null) {
- addrs.add(addr);
- }
+ if (addr != null) {
+ addrs.add(addr);
}
}
return addrs;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/76dc2f53/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilterInitializer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilterInitializer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilterInitializer.java
index bec62ce..63c2cf3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilterInitializer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilterInitializer.java
@@ -135,6 +135,15 @@ public class TestAmFilterInitializer extends TestCase {
assertEquals(WebAppUtils.getResolvedRMWebAppURLWithoutScheme(conf),
proxyHosts.get(0));
+ // Check conf in which only RM hostname is set
+ conf = new Configuration(false);
+ conf.set(YarnConfiguration.RM_WEBAPP_ADDRESS,
+ "${yarn.resourcemanager.hostname}:8088"); // default in yarn-default.xml
+ conf.set(YarnConfiguration.RM_HOSTNAME, "host1");
+ proxyHosts = WebAppUtils.getProxyHostsAndPortsForAmFilter(conf);
+ assertEquals(1, proxyHosts.size());
+ assertEquals("host1:8088", proxyHosts.get(0));
+
// Check PROXY_ADDRESS has priority
conf = new Configuration(false);
conf.set(YarnConfiguration.PROXY_ADDRESS, "host1:1000");
@@ -188,6 +197,44 @@ public class TestAmFilterInitializer extends TestCase {
Collections.sort(proxyHosts);
assertEquals("host5:5000", proxyHosts.get(0));
assertEquals("host6:6000", proxyHosts.get(1));
+
+ // Check config without explicit RM_WEBAPP_ADDRESS settings (RM HA)
+ conf = new Configuration(false);
+ conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
+ conf.set(YarnConfiguration.RM_HA_IDS, "rm1,rm2,rm3");
+ conf.set(YarnConfiguration.RM_HOSTNAME + ".rm1", "host2");
+ conf.set(YarnConfiguration.RM_HOSTNAME + ".rm2", "host3");
+ conf.set(YarnConfiguration.RM_HOSTNAME + ".rm3", "host4");
+ conf.set(YarnConfiguration.RM_HOSTNAME + ".rm4", "dummy");
+ proxyHosts = WebAppUtils.getProxyHostsAndPortsForAmFilter(conf);
+ assertEquals(3, proxyHosts.size());
+ Collections.sort(proxyHosts);
+ assertEquals("host2:" + YarnConfiguration.DEFAULT_RM_WEBAPP_PORT,
+ proxyHosts.get(0));
+ assertEquals("host3:" + YarnConfiguration.DEFAULT_RM_WEBAPP_PORT,
+ proxyHosts.get(1));
+ assertEquals("host4:" + YarnConfiguration.DEFAULT_RM_WEBAPP_PORT,
+ proxyHosts.get(2));
+
+ // Check config without explicit RM_WEBAPP_HTTPS_ADDRESS settings (RM HA)
+ conf = new Configuration(false);
+ conf.set(YarnConfiguration.YARN_HTTP_POLICY_KEY,
+ HttpConfig.Policy.HTTPS_ONLY.toString());
+ conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
+ conf.set(YarnConfiguration.RM_HA_IDS, "rm1,rm2,rm3");
+ conf.set(YarnConfiguration.RM_HOSTNAME + ".rm1", "host2");
+ conf.set(YarnConfiguration.RM_HOSTNAME + ".rm2", "host3");
+ conf.set(YarnConfiguration.RM_HOSTNAME + ".rm3", "host4");
+ conf.set(YarnConfiguration.RM_HOSTNAME + ".rm4", "dummy");
+ proxyHosts = WebAppUtils.getProxyHostsAndPortsForAmFilter(conf);
+ assertEquals(3, proxyHosts.size());
+ Collections.sort(proxyHosts);
+ assertEquals("host2:" + YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_PORT,
+ proxyHosts.get(0));
+ assertEquals("host3:" + YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_PORT,
+ proxyHosts.get(1));
+ assertEquals("host4:" + YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_PORT,
+ proxyHosts.get(2));
}
class MockAmFilterInitializer extends AmFilterInitializer {