You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by pb...@apache.org on 2021/04/01 07:21:41 UTC
[hadoop] branch trunk updated: YARN-10720. YARN WebAppProxyServlet
should support connection timeout to prevent proxy server from hanging.
Contributed by Qi Zhu.
This is an automated email from the ASF dual-hosted git repository.
pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new a0deda1 YARN-10720. YARN WebAppProxyServlet should support connection timeout to prevent proxy server from hanging. Contributed by Qi Zhu.
a0deda1 is described below
commit a0deda1a777d8967fb8c08ac976543cda895773d
Author: Peter Bacsko <pb...@cloudera.com>
AuthorDate: Thu Apr 1 09:21:15 2021 +0200
YARN-10720. YARN WebAppProxyServlet should support connection timeout to prevent proxy server from hanging. Contributed by Qi Zhu.
---
.../apache/hadoop/yarn/conf/YarnConfiguration.java | 14 ++++
.../src/main/resources/yarn-default.xml | 12 ++++
.../yarn/server/webproxy/WebAppProxyServlet.java | 28 ++++++--
.../server/webproxy/TestWebAppProxyServlet.java | 79 +++++++++++++++++++++-
4 files changed, 126 insertions(+), 7 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 2cf4a3b..2103ae7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -2668,6 +2668,20 @@ public class YarnConfiguration extends Configuration {
public static final String DEFAULT_RM_APPLICATION_HTTPS_POLICY = "NONE";
+
+ // If the proxy connection time enabled.
+ public static final String RM_PROXY_TIMEOUT_ENABLED =
+ RM_PREFIX + "proxy.timeout.enabled";
+
+ public static final boolean DEFALUT_RM_PROXY_TIMEOUT_ENABLED =
+ true;
+
+ public static final String RM_PROXY_CONNECTION_TIMEOUT =
+ RM_PREFIX + "proxy.connection.timeout";
+
+ public static final int DEFAULT_RM_PROXY_CONNECTION_TIMEOUT =
+ 60000;
+
/**
* Interval of time the linux container executor should try cleaning up
* cgroups entry when cleaning up a container. This is required due to what
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 12fc32f..c51cf8a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -2574,6 +2574,18 @@
<value/>
</property>
+ <property>
+ <description>Enable the web proxy connection timeout, default is enabled.</description>
+ <name>yarn.resourcemanager.proxy.timeout.enabled</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <description>The web proxy connection timeout.</description>
+ <name>yarn.resourcemanager.proxy.connection.timeout</name>
+ <value>60000</value>
+ </property>
+
<!-- Applications' Configuration -->
<property>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
index 0b6bb65..03b7077 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
@@ -122,6 +122,9 @@ public class WebAppProxyServlet extends HttpServlet {
}
}
+ protected void setConf(YarnConfiguration conf){
+ this.conf = conf;
+ }
/**
* Default constructor
*/
@@ -230,6 +233,14 @@ public class WebAppProxyServlet extends HttpServlet {
String httpsPolicy = conf.get(YarnConfiguration.RM_APPLICATION_HTTPS_POLICY,
YarnConfiguration.DEFAULT_RM_APPLICATION_HTTPS_POLICY);
+
+ boolean connectionTimeoutEnabled =
+ conf.getBoolean(YarnConfiguration.RM_PROXY_TIMEOUT_ENABLED,
+ YarnConfiguration.DEFALUT_RM_PROXY_TIMEOUT_ENABLED);
+ int connectionTimeout =
+ conf.getInt(YarnConfiguration.RM_PROXY_CONNECTION_TIMEOUT,
+ YarnConfiguration.DEFAULT_RM_PROXY_CONNECTION_TIMEOUT);
+
if (httpsPolicy.equals("LENIENT") || httpsPolicy.equals("STRICT")) {
ProxyCA proxyCA = getProxyCA();
// ProxyCA could be null when the Proxy is run outside the RM
@@ -250,10 +261,18 @@ public class WebAppProxyServlet extends HttpServlet {
InetAddress localAddress = InetAddress.getByName(proxyHost);
LOG.debug("local InetAddress for proxy host: {}", localAddress);
httpClientBuilder.setDefaultRequestConfig(
- RequestConfig.custom()
- .setCircularRedirectsAllowed(true)
- .setLocalAddress(localAddress)
- .build());
+ connectionTimeoutEnabled ?
+ RequestConfig.custom()
+ .setCircularRedirectsAllowed(true)
+ .setLocalAddress(localAddress)
+ .setConnectionRequestTimeout(connectionTimeout)
+ .setSocketTimeout(connectionTimeout)
+ .setConnectTimeout(connectionTimeout)
+ .build() :
+ RequestConfig.custom()
+ .setCircularRedirectsAllowed(true)
+ .setLocalAddress(localAddress)
+ .build());
HttpRequestBase base = null;
if (method.equals(HTTP.GET)) {
@@ -621,7 +640,6 @@ public class WebAppProxyServlet extends HttpServlet {
* again... If this method returns true, there was a redirect, and
* it was handled by redirecting the current request to an error page.
*
- * @param path the part of the request path after the app id
* @param id the app id
* @param req the request object
* @param resp the response object
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java
index f05e05a..6c8993f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java
@@ -23,6 +23,8 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
@@ -35,10 +37,14 @@ import java.net.HttpCookie;
import java.net.HttpURLConnection;
import java.net.URI;
import java.net.URL;
+import java.net.SocketTimeoutException;
+import java.util.Collections;
import java.util.Enumeration;
import java.util.List;
import java.util.Map;
+import javax.servlet.ServletConfig;
+import javax.servlet.ServletContext;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
@@ -98,6 +104,7 @@ public class TestWebAppProxyServlet {
context.setContextPath("/foo");
server.setHandler(context);
context.addServlet(new ServletHolder(TestServlet.class), "/bar");
+ context.addServlet(new ServletHolder(TimeOutTestServlet.class), "/timeout");
((ServerConnector)server.getConnectors()[0]).setHost("localhost");
server.start();
originalPort = ((ServerConnector)server.getConnectors()[0]).getLocalPort();
@@ -145,6 +152,29 @@ public class TestWebAppProxyServlet {
}
}
+ @SuppressWarnings("serial")
+ public static class TimeOutTestServlet extends HttpServlet {
+
+ @Override
+ protected void doGet(HttpServletRequest req, HttpServletResponse resp)
+ throws ServletException, IOException {
+ try {
+ Thread.sleep(10 * 1000);
+ } catch (InterruptedException e) {
+ LOG.warn("doGet() interrupted", e);
+ resp.setStatus(HttpServletResponse.SC_BAD_REQUEST);
+ return;
+ }
+ resp.setStatus(HttpServletResponse.SC_OK);
+ }
+
+ @Override
+ protected void doPost(HttpServletRequest req, HttpServletResponse resp)
+ throws ServletException, IOException {
+ resp.setStatus(HttpServletResponse.SC_OK);
+ }
+ }
+
@Test(timeout=5000)
public void testWebAppProxyServlet() throws Exception {
configuration.set(YarnConfiguration.PROXY_ADDRESS, "localhost:9090");
@@ -256,6 +286,45 @@ public class TestWebAppProxyServlet {
}
}
+ @Test(expected = SocketTimeoutException.class)
+ public void testWebAppProxyConnectionTimeout()
+ throws IOException, ServletException{
+ HttpServletRequest request = mock(HttpServletRequest.class);
+ when(request.getMethod()).thenReturn("GET");
+ when(request.getRemoteUser()).thenReturn("dr.who");
+ when(request.getPathInfo()).thenReturn("/application_00_0");
+ when(request.getHeaderNames()).thenReturn(Collections.emptyEnumeration());
+
+ HttpServletResponse response = mock(HttpServletResponse.class);
+ when(response.getOutputStream()).thenReturn(null);
+
+ WebAppProxyServlet servlet = new WebAppProxyServlet();
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.setBoolean(YarnConfiguration.RM_PROXY_TIMEOUT_ENABLED,
+ true);
+ conf.setInt(YarnConfiguration.RM_PROXY_CONNECTION_TIMEOUT,
+ 1000);
+
+ servlet.setConf(conf);
+
+ ServletConfig config = mock(ServletConfig.class);
+ ServletContext context = mock(ServletContext.class);
+ when(config.getServletContext()).thenReturn(context);
+
+ AppReportFetcherForTest appReportFetcher =
+ new AppReportFetcherForTest(new YarnConfiguration());
+
+ when(config.getServletContext()
+ .getAttribute(WebAppProxy.FETCHER_ATTRIBUTE))
+ .thenReturn(appReportFetcher);
+
+ appReportFetcher.answer = 7;
+
+ servlet.init(config);
+ servlet.doGet(request, response);
+
+ }
+
@Test(timeout=5000)
public void testAppReportForEmptyTrackingUrl() throws Exception {
configuration.set(YarnConfiguration.PROXY_ADDRESS, "localhost:9090");
@@ -391,9 +460,9 @@ public class TestWebAppProxyServlet {
@Test(timeout=5000)
public void testCheckHttpsStrictAndNotProvided() throws Exception {
- HttpServletResponse resp = Mockito.mock(HttpServletResponse.class);
+ HttpServletResponse resp = mock(HttpServletResponse.class);
StringWriter sw = new StringWriter();
- Mockito.when(resp.getWriter()).thenReturn(new PrintWriter(sw));
+ when(resp.getWriter()).thenReturn(new PrintWriter(sw));
YarnConfiguration conf = new YarnConfiguration();
final URI httpLink = new URI("http://foo.com");
final URI httpsLink = new URI("https://foo.com");
@@ -566,6 +635,12 @@ public class TestWebAppProxyServlet {
return result;
} else if (answer == 6) {
return getDefaultApplicationReport(appId, false);
+ } else if (answer == 7) {
+ // test connection timeout
+ FetchedAppReport result = getDefaultApplicationReport(appId);
+ result.getApplicationReport().setOriginalTrackingUrl("localhost:"
+ + originalPort + "/foo/timeout?a=b#main");
+ return result;
}
return null;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org