You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xy...@apache.org on 2017/07/19 22:35:11 UTC
[19/50] [abbrv] hadoop git commit: YARN-6625. yarn application -list
returns a tracking URL for AM that doesn't work in secured and HA
environment. (Yufei Gu)
YARN-6625. yarn application -list returns a tracking URL for AM that doesn't work in secured and HA environment. (Yufei Gu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9e0cde14
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9e0cde14
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9e0cde14
Branch: refs/heads/HDFS-7240
Commit: 9e0cde1469b8ffeb59619c64d6ece86b62424f04
Parents: e7d187a
Author: Yufei Gu <yu...@apache.org>
Authored: Fri Jul 14 14:10:45 2017 -0700
Committer: Yufei Gu <yu...@apache.org>
Committed: Fri Jul 14 14:10:45 2017 -0700
----------------------------------------------------------------------
.../server/webproxy/amfilter/AmIpFilter.java | 60 +++++++++++++----
.../server/webproxy/amfilter/TestAmFilter.java | 70 +++++++++++++++++++-
2 files changed, 114 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e0cde14/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java
index 6579191..cdab405 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java
@@ -23,10 +23,12 @@ import java.net.InetAddress;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.UnknownHostException;
+import java.net.HttpURLConnection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
+import java.util.Collection;
import javax.servlet.Filter;
import javax.servlet.FilterChain;
@@ -38,12 +40,12 @@ import javax.servlet.http.Cookie;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.yarn.conf.HAUtil;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.webproxy.ProxyUtils;
import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServlet;
-import org.apache.hadoop.yarn.util.RMHAUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -66,7 +68,8 @@ public class AmIpFilter implements Filter {
private String[] proxyHosts;
private Set<String> proxyAddresses = null;
private long lastUpdate;
- private Map<String, String> proxyUriBases;
+ @VisibleForTesting
+ Map<String, String> proxyUriBases;
@Override
public void init(FilterConfig conf) throws ServletException {
@@ -187,24 +190,55 @@ public class AmIpFilter implements Filter {
}
}
- protected String findRedirectUrl() throws ServletException {
- String addr;
- if (proxyUriBases.size() == 1) { // external proxy or not RM HA
+ @VisibleForTesting
+ public String findRedirectUrl() throws ServletException {
+ String addr = null;
+ if (proxyUriBases.size() == 1) {
+ // external proxy or not RM HA
addr = proxyUriBases.values().iterator().next();
- } else { // RM HA
+ } else {
+ // RM HA
YarnConfiguration conf = new YarnConfiguration();
- String activeRMId = RMHAUtils.findActiveRMHAId(conf);
- String addressPropertyPrefix = YarnConfiguration.useHttps(conf)
- ? YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS
- : YarnConfiguration.RM_WEBAPP_ADDRESS;
- String host = conf.get(
- HAUtil.addSuffix(addressPropertyPrefix, activeRMId));
- addr = proxyUriBases.get(host);
+ for (String rmId : getRmIds(conf)) {
+ String url = getUrlByRmId(conf, rmId);
+ if (isValidUrl(url)) {
+ addr = url;
+ break;
+ }
+ }
}
+
if (addr == null) {
throw new ServletException(
"Could not determine the proxy server for redirection");
}
return addr;
}
+
+ @VisibleForTesting
+ Collection<String> getRmIds(YarnConfiguration conf) {
+ return conf.getStringCollection(YarnConfiguration.RM_HA_IDS);
+ }
+
+ @VisibleForTesting
+ String getUrlByRmId(YarnConfiguration conf, String rmId) {
+ String addressPropertyPrefix = YarnConfiguration.useHttps(conf) ?
+ YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS :
+ YarnConfiguration.RM_WEBAPP_ADDRESS;
+ String host = conf.get(HAUtil.addSuffix(addressPropertyPrefix, rmId));
+ return proxyUriBases.get(host);
+ }
+
+ private boolean isValidUrl(String url) {
+ boolean isValid = false;
+ try {
+ HttpURLConnection conn =
+ (HttpURLConnection) new URL(url).openConnection();
+ conn.connect();
+ isValid = conn.getResponseCode() == HttpURLConnection.HTTP_OK;
+ } catch (Exception e) {
+ LOG.debug("Failed to connect to " + url + ": " + e.toString());
+ }
+ return isValid;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e0cde14/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilter.java
index b788f5d..687faea 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilter.java
@@ -22,18 +22,41 @@ import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.net.HttpURLConnection;
-import java.util.*;
+import java.util.Set;
+import java.util.HashSet;
+import java.util.Enumeration;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.ArrayList;
+import java.util.Arrays;
import java.util.concurrent.atomic.AtomicBoolean;
-import javax.servlet.*;
+import javax.servlet.FilterConfig;
+import javax.servlet.FilterChain;
+import javax.servlet.Filter;
+import javax.servlet.ServletContext;
+import javax.servlet.ServletResponse;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletException;
import javax.servlet.http.Cookie;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+import org.apache.hadoop.http.TestHttpServer;
import org.apache.hadoop.yarn.server.webproxy.ProxyUtils;
import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServlet;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.server.ServerConnector;
+import org.eclipse.jetty.servlet.ServletContextHandler;
+import org.eclipse.jetty.servlet.ServletHolder;
+import org.eclipse.jetty.util.thread.QueuedThreadPool;
import org.glassfish.grizzly.servlet.HttpServletResponseImpl;
import org.junit.Test;
import org.mockito.Mockito;
@@ -121,6 +144,47 @@ public class TestAmFilter {
filter.destroy();
}
+ @Test
+ public void testFindRedirectUrl() throws Exception {
+ final String rm1 = "rm1";
+ final String rm2 = "rm2";
+ // generate a valid URL
+ final String rm1Url = startHttpServer();
+ // invalid url
+ final String rm2Url = "host2:8088";
+
+ TestAmIpFilter filter = new TestAmIpFilter();
+ TestAmIpFilter spy = Mockito.spy(filter);
+ // make sure findRedirectUrl() go to HA branch
+ spy.proxyUriBases = new HashMap<>();
+ spy.proxyUriBases.put(rm1, rm1Url);
+ spy.proxyUriBases.put(rm2, rm2Url);
+
+ Collection<String> rmIds = new ArrayList<>(Arrays.asList(rm1, rm2));
+ Mockito.doReturn(rmIds).when(spy).getRmIds(Mockito.any());
+ Mockito.doReturn(rm1Url).when(spy)
+ .getUrlByRmId(Mockito.any(), Mockito.eq(rm2));
+ Mockito.doReturn(rm2Url).when(spy)
+ .getUrlByRmId(Mockito.any(), Mockito.eq(rm1));
+
+ assertEquals(spy.findRedirectUrl(), rm1Url);
+ }
+
+ private String startHttpServer() throws Exception {
+ Server server = new Server(0);
+ ((QueuedThreadPool)server.getThreadPool()).setMaxThreads(10);
+ ServletContextHandler context = new ServletContextHandler();
+ context.setContextPath("/foo");
+ server.setHandler(context);
+ String servletPath = "/bar";
+ context.addServlet(new ServletHolder(TestHttpServer.EchoServlet.class),
+ servletPath);
+ ((ServerConnector)server.getConnectors()[0]).setHost("localhost");
+ server.start();
+ System.setProperty("sun.net.http.allowRestrictedHeaders", "true");
+ return server.getURI().toString() + servletPath;
+ }
+
/**
* Test AmIpFilter
*/
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org