You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@knox.apache.org by km...@apache.org on 2016/03/18 13:50:01 UTC
knox git commit: [KNOX-695] - Expose configuration of HttpClient's
connection and socket timeout settings
Repository: knox
Updated Branches:
refs/heads/master 15ae0d781 -> 8cb451daf
[KNOX-695] - Expose configuration of HttpClient's connection and socket timeout settings
Project: http://git-wip-us.apache.org/repos/asf/knox/repo
Commit: http://git-wip-us.apache.org/repos/asf/knox/commit/8cb451da
Tree: http://git-wip-us.apache.org/repos/asf/knox/tree/8cb451da
Diff: http://git-wip-us.apache.org/repos/asf/knox/diff/8cb451da
Branch: refs/heads/master
Commit: 8cb451daf2d2db841780eab78afe928de397c966
Parents: 15ae0d7
Author: Kevin Minder <km...@apache.org>
Authored: Fri Mar 18 08:49:57 2016 -0400
Committer: Kevin Minder <km...@apache.org>
Committed: Fri Mar 18 08:49:57 2016 -0400
----------------------------------------------------------------------
CHANGES | 1 +
.../gateway/config/impl/GatewayConfigImpl.java | 40 +++++++++++
gateway-spi/pom.xml | 4 ++
.../hadoop/gateway/config/GatewayConfig.java | 4 ++
.../gateway/dispatch/DefaultDispatch.java | 6 +-
.../dispatch/DefaultHttpClientFactory.java | 74 ++++++++++++++++++--
.../hadoop/gateway/GatewayTestConfig.java | 10 +++
.../hadoop/gateway/GatewayTestConfig.java | 10 +++
8 files changed, 142 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/knox/blob/8cb451da/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 5963fca..6d21512 100644
--- a/CHANGES
+++ b/CHANGES
@@ -11,6 +11,7 @@ Release Notes - Apache Knox - Version 0.9.0
* [KNOX-678] - Malformed UTF-8 characters in JSON Response
* [KNOX-680] - Update Knox's HttpClient dependency to latest version
* [KNOX-694] - Enhance LDAP user search configurability
+ * [KNOX-695] - Expose configuration of HttpClient's connection and socket timeout settings
** Bug
* [KNOX-681] - A PUT with Content-Type application/xml but no body causes NullPointerException
http://git-wip-us.apache.org/repos/asf/knox/blob/8cb451da/gateway-server/src/main/java/org/apache/hadoop/gateway/config/impl/GatewayConfigImpl.java
----------------------------------------------------------------------
diff --git a/gateway-server/src/main/java/org/apache/hadoop/gateway/config/impl/GatewayConfigImpl.java b/gateway-server/src/main/java/org/apache/hadoop/gateway/config/impl/GatewayConfigImpl.java
index 711efc8..82a21ee 100644
--- a/gateway-server/src/main/java/org/apache/hadoop/gateway/config/impl/GatewayConfigImpl.java
+++ b/gateway-server/src/main/java/org/apache/hadoop/gateway/config/impl/GatewayConfigImpl.java
@@ -119,6 +119,8 @@ public class GatewayConfigImpl extends Configuration implements GatewayConfig {
private static final String XFORWARDED_ENABLED = GATEWAY_CONFIG_FILE_PREFIX + ".xforwarded.enabled";
private static final String EPHEMERAL_DH_KEY_SIZE = GATEWAY_CONFIG_FILE_PREFIX + ".jdk.tls.ephemeralDHKeySize";
private static final String HTTP_CLIENT_MAX_CONNECTION = GATEWAY_CONFIG_FILE_PREFIX + ".httpclient.maxConnections";
+ private static final String HTTP_CLIENT_CONNECTION_TIMEOUT = GATEWAY_CONFIG_FILE_PREFIX + ".httpclient.connectionTimeout";
+ private static final String HTTP_CLIENT_SOCKET_TIMEOUT = GATEWAY_CONFIG_FILE_PREFIX + ".httpclient.socketTimeout";
private static final String THREAD_POOL_MAX = GATEWAY_CONFIG_FILE_PREFIX + ".threadpool.max";
public static final String HTTP_SERVER_REQUEST_BUFFER = GATEWAY_CONFIG_FILE_PREFIX + ".httpserver.requestBuffer";
public static final String HTTP_SERVER_REQUEST_HEADER_BUFFER = GATEWAY_CONFIG_FILE_PREFIX + ".httpserver.requestHeaderBuffer";
@@ -500,6 +502,34 @@ public class GatewayConfigImpl extends Configuration implements GatewayConfig {
return getInt( HTTP_CLIENT_MAX_CONNECTION, 32 );
}
+ @Override
+ public int getHttpClientConnectionTimeout() {
+ int t = -1;
+ String s = get( HTTP_CLIENT_CONNECTION_TIMEOUT, null );
+ if ( s != null ) {
+ try {
+ t = (int)parseNetworkTimeout( s );
+ } catch ( Exception e ) {
+ // Ignore it and use the default.
+ }
+ }
+ return t;
+ }
+
+ @Override
+ public int getHttpClientSocketTimeout() {
+ int t = -1;
+ String s = get( HTTP_CLIENT_SOCKET_TIMEOUT, null );
+ if ( s != null ) {
+ try {
+ t = (int)parseNetworkTimeout( s );
+ } catch ( Exception e ) {
+ // Ignore it and use the default.
+ }
+ }
+ return t;
+ }
+
/* (non-Javadoc)
* @see org.apache.hadoop.gateway.config.GatewayConfig#getThreadPoolMax()
*/
@@ -572,4 +602,14 @@ public class GatewayConfigImpl extends Configuration implements GatewayConfig {
public String getSigningKeyAlias() {
return get(SIGNING_KEY_ALIAS);
}
+
+ private static long parseNetworkTimeout( String s ) {
+ PeriodFormatter f = new PeriodFormatterBuilder()
+ .appendMinutes().appendSuffix("m"," min")
+ .appendSeconds().appendSuffix("s"," sec")
+ .appendMillis().toFormatter();
+ Period p = Period.parse( s, f );
+ return p.toStandardDuration().getMillis();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/knox/blob/8cb451da/gateway-spi/pom.xml
----------------------------------------------------------------------
diff --git a/gateway-spi/pom.xml b/gateway-spi/pom.xml
index 196515e..1da99d9 100644
--- a/gateway-spi/pom.xml
+++ b/gateway-spi/pom.xml
@@ -110,6 +110,10 @@
<groupId>commons-collections</groupId>
<artifactId>commons-collections</artifactId>
</dependency>
+ <dependency>
+ <groupId>joda-time</groupId>
+ <artifactId>joda-time</artifactId>
+ </dependency>
<dependency>
<groupId>junit</groupId>
http://git-wip-us.apache.org/repos/asf/knox/blob/8cb451da/gateway-spi/src/main/java/org/apache/hadoop/gateway/config/GatewayConfig.java
----------------------------------------------------------------------
diff --git a/gateway-spi/src/main/java/org/apache/hadoop/gateway/config/GatewayConfig.java b/gateway-spi/src/main/java/org/apache/hadoop/gateway/config/GatewayConfig.java
index 5558578..d830887 100644
--- a/gateway-spi/src/main/java/org/apache/hadoop/gateway/config/GatewayConfig.java
+++ b/gateway-spi/src/main/java/org/apache/hadoop/gateway/config/GatewayConfig.java
@@ -121,6 +121,10 @@ public interface GatewayConfig {
int getHttpClientMaxConnections();
+ int getHttpClientConnectionTimeout();
+
+ int getHttpClientSocketTimeout();
+
int getThreadPoolMax();
int getHttpServerRequestBuffer();
http://git-wip-us.apache.org/repos/asf/knox/blob/8cb451da/gateway-spi/src/main/java/org/apache/hadoop/gateway/dispatch/DefaultDispatch.java
----------------------------------------------------------------------
diff --git a/gateway-spi/src/main/java/org/apache/hadoop/gateway/dispatch/DefaultDispatch.java b/gateway-spi/src/main/java/org/apache/hadoop/gateway/dispatch/DefaultDispatch.java
index d1cef55..6c72fde 100644
--- a/gateway-spi/src/main/java/org/apache/hadoop/gateway/dispatch/DefaultDispatch.java
+++ b/gateway-spi/src/main/java/org/apache/hadoop/gateway/dispatch/DefaultDispatch.java
@@ -254,9 +254,11 @@ public class DefaultDispatch extends AbstractGatewayDispatch {
@Override
public void doGet(URI url, HttpServletRequest request, HttpServletResponse response)
throws IOException, URISyntaxException {
- HttpGet method = new HttpGet(url);
+ HttpGet method = new HttpGet(url);
// https://issues.apache.org/jira/browse/KNOX-107 - Service URLs not rewritten for WebHDFS GET redirects
- method.getParams().setBooleanParameter("http.protocol.handle-redirects", false);
+ // This is now taken care of in DefaultHttpClientFactory.createHttpClient
+ // and setting params here causes configuration setup there to be ignored there.
+ // method.getParams().setBooleanParameter("http.protocol.handle-redirects", false);
copyRequestHeaderFields(method, request);
executeRequest(method, request, response);
}
http://git-wip-us.apache.org/repos/asf/knox/blob/8cb451da/gateway-spi/src/main/java/org/apache/hadoop/gateway/dispatch/DefaultHttpClientFactory.java
----------------------------------------------------------------------
diff --git a/gateway-spi/src/main/java/org/apache/hadoop/gateway/dispatch/DefaultHttpClientFactory.java b/gateway-spi/src/main/java/org/apache/hadoop/gateway/dispatch/DefaultHttpClientFactory.java
index 57de2e4..92652e2 100644
--- a/gateway-spi/src/main/java/org/apache/hadoop/gateway/dispatch/DefaultHttpClientFactory.java
+++ b/gateway-spi/src/main/java/org/apache/hadoop/gateway/dispatch/DefaultHttpClientFactory.java
@@ -30,17 +30,20 @@ import org.apache.http.client.HttpClient;
import org.apache.http.client.HttpRequestRetryHandler;
import org.apache.http.client.RedirectStrategy;
import org.apache.http.client.config.AuthSchemes;
+import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.config.Registry;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.cookie.Cookie;
import org.apache.http.impl.DefaultConnectionReuseStrategy;
-import org.apache.http.impl.auth.SPNegoSchemeFactory;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.DefaultConnectionKeepAliveStrategy;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.protocol.HttpContext;
+import org.joda.time.Period;
+import org.joda.time.format.PeriodFormatter;
+import org.joda.time.format.PeriodFormatterBuilder;
import javax.servlet.FilterConfig;
import java.io.IOException;
@@ -72,15 +75,31 @@ public class DefaultHttpClientFactory implements HttpClientFactory {
builder.setKeepAliveStrategy( DefaultConnectionKeepAliveStrategy.INSTANCE );
builder.setConnectionReuseStrategy( DefaultConnectionReuseStrategy.INSTANCE );
+ builder.setRedirectStrategy( new NeverRedirectStrategy() );
+ builder.setRetryHandler( new NeverRetryHandler() );
int maxConnections = getMaxConnections( filterConfig );
builder.setMaxConnTotal( maxConnections );
builder.setMaxConnPerRoute( maxConnections );
- return builder
- .setRedirectStrategy(new NeverRedirectStrategy())
- .setRetryHandler(new NeverRetryHandler())
- .build();
+ builder.setDefaultRequestConfig( getRequestConfig( filterConfig ) );
+
+ HttpClient client = builder.build();
+ return client;
+ }
+
+ private static RequestConfig getRequestConfig( FilterConfig config ) {
+ RequestConfig.Builder builder = RequestConfig.custom();
+ int connectionTimeout = getConnectionTimeout( config );
+ if ( connectionTimeout != -1 ) {
+ builder.setConnectTimeout( connectionTimeout );
+ builder.setConnectionRequestTimeout( connectionTimeout );
+ }
+ int socketTimeout = getSocketTimeout( config );
+ if( socketTimeout != -1 ) {
+ builder.setSocketTimeout( socketTimeout );
+ }
+ return builder.build();
}
private class NoCookieStore implements CookieStore {
@@ -156,4 +175,49 @@ public class DefaultHttpClientFactory implements HttpClientFactory {
return maxConnections;
}
+ private static int getConnectionTimeout( FilterConfig filterConfig ) {
+ int timeout = -1;
+ GatewayConfig globalConfig =
+ (GatewayConfig)filterConfig.getServletContext().getAttribute( GatewayConfig.GATEWAY_CONFIG_ATTRIBUTE );
+ if( globalConfig != null ) {
+ timeout = globalConfig.getHttpClientConnectionTimeout();
+ }
+ String str = filterConfig.getInitParameter( "httpclient.connectionTimeout" );
+ if( str != null ) {
+ try {
+ timeout = (int)parseTimeout( str );
+ } catch ( Exception e ) {
+ // Ignore it and use the default.
+ }
+ }
+ return timeout;
+ }
+
+ private static int getSocketTimeout( FilterConfig filterConfig ) {
+ int timeout = -1;
+ GatewayConfig globalConfig =
+ (GatewayConfig)filterConfig.getServletContext().getAttribute( GatewayConfig.GATEWAY_CONFIG_ATTRIBUTE );
+ if( globalConfig != null ) {
+ timeout = globalConfig.getHttpClientSocketTimeout();
+ }
+ String str = filterConfig.getInitParameter( "httpclient.socketTimeout" );
+ if( str != null ) {
+ try {
+ timeout = (int)parseTimeout( str );
+ } catch ( Exception e ) {
+ // Ignore it and use the default.
+ }
+ }
+ return timeout;
+ }
+
+ private static long parseTimeout( String s ) {
+ PeriodFormatter f = new PeriodFormatterBuilder()
+ .appendMinutes().appendSuffix("m"," min")
+ .appendSeconds().appendSuffix("s"," sec")
+ .appendMillis().toFormatter();
+ Period p = Period.parse( s, f );
+ return p.toStandardDuration().getMillis();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/knox/blob/8cb451da/gateway-test-release/webhdfs-kerb-test/src/test/java/org/apache/hadoop/gateway/GatewayTestConfig.java
----------------------------------------------------------------------
diff --git a/gateway-test-release/webhdfs-kerb-test/src/test/java/org/apache/hadoop/gateway/GatewayTestConfig.java b/gateway-test-release/webhdfs-kerb-test/src/test/java/org/apache/hadoop/gateway/GatewayTestConfig.java
index e5be4be..3c53597 100644
--- a/gateway-test-release/webhdfs-kerb-test/src/test/java/org/apache/hadoop/gateway/GatewayTestConfig.java
+++ b/gateway-test-release/webhdfs-kerb-test/src/test/java/org/apache/hadoop/gateway/GatewayTestConfig.java
@@ -285,6 +285,16 @@ public class GatewayTestConfig extends Configuration implements GatewayConfig {
}
@Override
+ public int getHttpClientConnectionTimeout() {
+ return -1;
+ }
+
+ @Override
+ public int getHttpClientSocketTimeout() {
+ return -1;
+ }
+
+ @Override
public int getThreadPoolMax() {
return 16;
}
http://git-wip-us.apache.org/repos/asf/knox/blob/8cb451da/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayTestConfig.java
----------------------------------------------------------------------
diff --git a/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayTestConfig.java b/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayTestConfig.java
index cd71e7f..4c75cb9 100644
--- a/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayTestConfig.java
+++ b/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayTestConfig.java
@@ -326,6 +326,16 @@ public class GatewayTestConfig extends Configuration implements GatewayConfig {
}
@Override
+ public int getHttpClientConnectionTimeout() {
+ return -1;
+ }
+
+ @Override
+ public int getHttpClientSocketTimeout() {
+ return -1;
+ }
+
+ @Override
public int getThreadPoolMax() {
return 16;
}