You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2015/05/19 12:14:38 UTC
[2/6] incubator-slider git commit: SLIDER-878 use own connection
factory with controlled order on connection operations
SLIDER-878 use own connection factory with controlled order on connection operations
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/1cecc3fc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/1cecc3fc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/1cecc3fc
Branch: refs/heads/develop
Commit: 1cecc3fcb0e80de48e0ea7ba54e607421ffea3fe
Parents: cac169a
Author: Steve Loughran <st...@apache.org>
Authored: Mon May 18 20:36:16 2015 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Mon May 18 20:36:16 2015 +0100
----------------------------------------------------------------------
pom.xml | 7 +-
.../restclient/SliderURLConnectionFactory.java | 176 +++++++++++++++++++
.../core/restclient/UgiJerseyBinding.java | 3 +-
.../restclient/UrlConnectionOperations.java | 25 +--
4 files changed, 186 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cecc3fc/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 6a9152c..53b948d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1591,7 +1591,7 @@
<!-- hadoop branch-2.7 builds -->
<id>branch-2.7</id>
<properties>
- <hadoop.version>2.7.0-SNAPSHOT</hadoop.version>
+ <hadoop.version>2.7.1-SNAPSHOT</hadoop.version>
</properties>
</profile>
@@ -1615,7 +1615,10 @@
<!-- Java 8 build -->
<id>java8</id>
<properties>
- <project.java.src.version>1.8</project.java.src.version>
+ <project.java.src.version>1.8</project.java.src.version>
+<!--
+ <jersey.version>1.19</jersey.version>
+-->
</properties>
</profile>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cecc3fc/slider-core/src/main/java/org/apache/slider/core/restclient/SliderURLConnectionFactory.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/restclient/SliderURLConnectionFactory.java b/slider-core/src/main/java/org/apache/slider/core/restclient/SliderURLConnectionFactory.java
new file mode 100644
index 0000000..e453f52
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/restclient/SliderURLConnectionFactory.java
@@ -0,0 +1,176 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.restclient;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.web.KerberosUgiAuthenticator;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.apache.hadoop.security.authentication.client.ConnectionConfigurator;
+import org.apache.hadoop.security.ssl.SSLFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.net.ssl.HostnameVerifier;
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLSocketFactory;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.net.URLConnection;
+import java.security.GeneralSecurityException;
+
+/**
+ * Factory for URL connections; used behind the scenes in the Jersey integration.
+ * <p>
+ * Derived from the WebHDFS implementation.
+ */
+public class SliderURLConnectionFactory {
+ private static final Logger log =
+ LoggerFactory.getLogger(SliderURLConnectionFactory.class);
+
+ /**
+ * Timeout for socket connects and reads
+ */
+ public final static int DEFAULT_SOCKET_TIMEOUT = 60 * 1000; // 1 minute
+ private final ConnectionConfigurator connConfigurator;
+
+ private static final ConnectionConfigurator DEFAULT_CONFIGURATOR = new BasicConfigurator();
+
+ /**
+ * Construct a new URLConnectionFactory based on the configuration. It will
+ * try to load SSL certificates when it is specified.
+ */
+ public static SliderURLConnectionFactory newInstance(Configuration conf) {
+ ConnectionConfigurator conn;
+ try {
+ conn = newSslConnConfigurator(DEFAULT_SOCKET_TIMEOUT, conf);
+ } catch (Exception e) {
+ log.debug("Cannot load customized SSL configuration.", e);
+ conn = DEFAULT_CONFIGURATOR;
+ }
+ return new SliderURLConnectionFactory(conn);
+ }
+
+ private SliderURLConnectionFactory(ConnectionConfigurator connConfigurator) {
+ this.connConfigurator = connConfigurator;
+ }
+
+ /**
+ * Create a new ConnectionConfigurator for SSL connections
+ */
+ private static ConnectionConfigurator newSslConnConfigurator(final int timeout,
+ Configuration conf) throws IOException, GeneralSecurityException {
+ final SSLFactory factory;
+ final SSLSocketFactory sf;
+ final HostnameVerifier hv;
+
+ factory = new SSLFactory(SSLFactory.Mode.CLIENT, conf);
+ factory.init();
+ sf = factory.createSSLSocketFactory();
+ hv = factory.getHostnameVerifier();
+
+ return new ConnectionConfigurator() {
+ @Override
+ public HttpURLConnection configure(HttpURLConnection conn)
+ throws IOException {
+ if (conn instanceof HttpsURLConnection) {
+ HttpsURLConnection c = (HttpsURLConnection) conn;
+ c.setSSLSocketFactory(sf);
+ c.setHostnameVerifier(hv);
+ }
+ SliderURLConnectionFactory.setupConnection(conn, timeout);
+ return conn;
+ }
+ };
+ }
+
+ /**
+ * Opens a url with read and connect timeouts
+ *
+ * @param url
+ * to open
+ * @return URLConnection
+ * @throws IOException
+ */
+ public URLConnection openConnection(URL url) throws IOException {
+ try {
+ return openConnection(url, false);
+ } catch (AuthenticationException e) {
+ // Unreachable
+ return null;
+ }
+ }
+
+ /**
+ * Opens a url with read and connect timeouts
+ *
+ * @param url
+ * URL to open
+ * @param isSpnego
+ * whether the url should be authenticated via SPNEGO
+ * @return URLConnection
+ * @throws IOException
+ * @throws AuthenticationException
+ */
+ public URLConnection openConnection(URL url, boolean isSpnego)
+ throws IOException, AuthenticationException {
+ if (isSpnego) {
+ log.debug("open AuthenticatedURL connection {}", url);
+ UserGroupInformation.getCurrentUser().checkTGTAndReloginFromKeytab();
+ final AuthenticatedURL.Token authToken = new AuthenticatedURL.Token();
+ return new AuthenticatedURL(new KerberosUgiAuthenticator(),
+ connConfigurator).openConnection(url, authToken);
+ } else {
+ log.debug("open URL connection {}", url);
+ URLConnection connection = url.openConnection();
+ if (connection instanceof HttpURLConnection) {
+ connConfigurator.configure((HttpURLConnection) connection);
+ }
+ return connection;
+ }
+ }
+
+ /**
+ * Sets connection parameters on the given URLConnection
+ *
+ * @param connection
+ * URLConnection to set
+ * @param socketTimeout
+ * the connection and read timeout of the connection.
+ */
+ private static void setupConnection(URLConnection connection, int socketTimeout) {
+ connection.setConnectTimeout(socketTimeout);
+ connection.setReadTimeout(socketTimeout);
+ connection.setUseCaches(false);
+ if (connection instanceof HttpURLConnection) {
+ ((HttpURLConnection) connection).setInstanceFollowRedirects(true);
+ }
+ }
+
+ private static class BasicConfigurator implements ConnectionConfigurator {
+ @Override
+ public HttpURLConnection configure(HttpURLConnection conn)
+ throws IOException {
+ SliderURLConnectionFactory.setupConnection(conn, DEFAULT_SOCKET_TIMEOUT);
+ return conn;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cecc3fc/slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java b/slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java
index 2526684..d19c621 100644
--- a/slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java
+++ b/slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java
@@ -88,8 +88,7 @@ public class UgiJerseyBinding implements
try {
// open a connection handling status codes and so redirections
// but as it opens a connection, it's less useful than you think.
-// return operations.openConnectionRedirecting(url);
-
+
return operations.openConnection(url);
} catch (AuthenticationException e) {
throw new IOException(e);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cecc3fc/slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java b/slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java
index 49fe5ea..e0d2315 100644
--- a/slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java
+++ b/slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java
@@ -22,7 +22,6 @@ import com.google.common.base.Preconditions;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.hdfs.web.URLConnectionFactory;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authentication.client.AuthenticationException;
@@ -31,24 +30,21 @@ import org.apache.hadoop.yarn.webapp.NotFoundException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.servlet.http.HttpServletResponse;
-import javax.ws.rs.core.HttpHeaders;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.HttpURLConnection;
-import java.net.URI;
import java.net.URL;
/**
- * Operations on the JDK UrlConnection class. This uses WebHDFS
- * methods to set up the operations.
+ * Operations on the JDK UrlConnection class.
+ *
*/
public class UrlConnectionOperations extends Configured {
private static final Logger log =
LoggerFactory.getLogger(UrlConnectionOperations.class);
- private URLConnectionFactory connectionFactory;
+ private SliderURLConnectionFactory connectionFactory;
private boolean useSpnego = false;
@@ -59,8 +55,7 @@ public class UrlConnectionOperations extends Configured {
*/
public UrlConnectionOperations(Configuration conf) {
super(conf);
- connectionFactory = URLConnectionFactory
- .newDefaultURLConnectionFactory(conf);
+ connectionFactory = SliderURLConnectionFactory.newInstance(conf);
if (UserGroupInformation.isSecurityEnabled()) {
log.debug("SPNEGO is enabled");
setUseSpnego(true);
@@ -88,18 +83,6 @@ public class UrlConnectionOperations extends Configured {
public HttpURLConnection openConnection(URL url) throws
IOException,
AuthenticationException {
-
- HttpURLConnection conn = innerOpenConnection(url);
- conn.setUseCaches(false);
- conn.setInstanceFollowRedirects(true);
- return conn;
- }
-
-
-
- protected HttpURLConnection innerOpenConnection(URL url) throws
- IOException,
- AuthenticationException {
Preconditions.checkArgument(url.getPort() != 0, "no port");
return (HttpURLConnection) connectionFactory.openConnection(url, useSpnego);
}