You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2015/05/19 12:14:37 UTC
[1/6] incubator-slider git commit: SLIDER-878 java 8 & jersey
compatibility
Repository: incubator-slider
Updated Branches:
refs/heads/develop 968299096 -> e94aa2696
SLIDER-878 java 8 & jersey compatibility
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/cac169a0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/cac169a0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/cac169a0
Branch: refs/heads/develop
Commit: cac169a02aa3bbaf1eaf7bb09981e00fd573c57d
Parents: ae58d43
Author: Steve Loughran <st...@apache.org>
Authored: Mon May 18 18:53:43 2015 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Mon May 18 18:53:43 2015 +0100
----------------------------------------------------------------------
pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/cac169a0/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index ca12be2..6a9152c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1615,7 +1615,7 @@
<!-- Java 8 build -->
<id>java8</id>
<properties>
- <project.java.src.version>8</project.java.src.version>
+ <project.java.src.version>1.8</project.java.src.version>
</properties>
</profile>
[5/6] incubator-slider git commit: Merge branch
'feature/SLIDER-878-jersey' into develop
Posted by st...@apache.org.
Merge branch 'feature/SLIDER-878-jersey' into develop
# Conflicts:
# pom.xml
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/36169512
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/36169512
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/36169512
Branch: refs/heads/develop
Commit: 3616951225067932d62ad5642f26e395f3792450
Parents: 9682990 f1efe76
Author: Steve Loughran <st...@apache.org>
Authored: Tue May 19 11:13:43 2015 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue May 19 11:13:43 2015 +0100
----------------------------------------------------------------------
.../restclient/SliderURLConnectionFactory.java | 176 +++++++++++++++++++
.../core/restclient/UgiJerseyBinding.java | 3 +-
.../restclient/UrlConnectionOperations.java | 25 +--
3 files changed, 181 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
[6/6] incubator-slider git commit: SLIDER-884 separate release-2.7
and branch-2.7 branches
Posted by st...@apache.org.
SLIDER-884 separate release-2.7 and branch-2.7 branches
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/e94aa269
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/e94aa269
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/e94aa269
Branch: refs/heads/develop
Commit: e94aa2696a2d20138fab09064aadd20ae224f6db
Parents: 3616951
Author: Steve Loughran <st...@apache.org>
Authored: Tue May 19 11:14:19 2015 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue May 19 11:14:19 2015 +0100
----------------------------------------------------------------------
pom.xml | 17 ++++++++++++++---
1 file changed, 14 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e94aa269/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index fa81da4..6a2f850 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1580,7 +1580,7 @@
</profile>
<profile>
- <!-- 2.6 snapshots -->
+ <!-- 2.6 -->
<id>release-2.6</id>
<properties>
<hadoop.version>2.6.0</hadoop.version>
@@ -1588,10 +1588,18 @@
</profile>
<profile>
+ <!-- 2.7.0 -->
+ <id>release-2.7</id>
+ <properties>
+ <hadoop.version>2.7.0</hadoop.version>
+ </properties>
+ </profile>
+
+ <profile>
<!-- hadoop branch-2.7 builds -->
<id>branch-2.7</id>
<properties>
- <hadoop.version>2.7.0</hadoop.version>
+ <hadoop.version>2.7.1-SNAPSHOT</hadoop.version>
</properties>
</profile>
@@ -1615,7 +1623,10 @@
<!-- Java 8 build -->
<id>java8</id>
<properties>
- <project.java.src.version>8</project.java.src.version>
+ <project.java.src.version>1.8</project.java.src.version>
+<!--
+ <jersey.version>1.19</jersey.version>
+-->
</properties>
</profile>
[2/6] incubator-slider git commit: SLIDER-878 use own connection
factory with controlled order on connection operations
Posted by st...@apache.org.
SLIDER-878 use own connection factory with controlled order on connection operations
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/1cecc3fc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/1cecc3fc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/1cecc3fc
Branch: refs/heads/develop
Commit: 1cecc3fcb0e80de48e0ea7ba54e607421ffea3fe
Parents: cac169a
Author: Steve Loughran <st...@apache.org>
Authored: Mon May 18 20:36:16 2015 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Mon May 18 20:36:16 2015 +0100
----------------------------------------------------------------------
pom.xml | 7 +-
.../restclient/SliderURLConnectionFactory.java | 176 +++++++++++++++++++
.../core/restclient/UgiJerseyBinding.java | 3 +-
.../restclient/UrlConnectionOperations.java | 25 +--
4 files changed, 186 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cecc3fc/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 6a9152c..53b948d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1591,7 +1591,7 @@
<!-- hadoop branch-2.7 builds -->
<id>branch-2.7</id>
<properties>
- <hadoop.version>2.7.0-SNAPSHOT</hadoop.version>
+ <hadoop.version>2.7.1-SNAPSHOT</hadoop.version>
</properties>
</profile>
@@ -1615,7 +1615,10 @@
<!-- Java 8 build -->
<id>java8</id>
<properties>
- <project.java.src.version>1.8</project.java.src.version>
+ <project.java.src.version>1.8</project.java.src.version>
+<!--
+ <jersey.version>1.19</jersey.version>
+-->
</properties>
</profile>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cecc3fc/slider-core/src/main/java/org/apache/slider/core/restclient/SliderURLConnectionFactory.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/restclient/SliderURLConnectionFactory.java b/slider-core/src/main/java/org/apache/slider/core/restclient/SliderURLConnectionFactory.java
new file mode 100644
index 0000000..e453f52
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/restclient/SliderURLConnectionFactory.java
@@ -0,0 +1,176 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.restclient;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.web.KerberosUgiAuthenticator;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.apache.hadoop.security.authentication.client.ConnectionConfigurator;
+import org.apache.hadoop.security.ssl.SSLFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.net.ssl.HostnameVerifier;
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLSocketFactory;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.net.URLConnection;
+import java.security.GeneralSecurityException;
+
+/**
+ * Factory for URL connections; used behind the scenes in the Jersey integration.
+ * <p>
+ * Derived from the WebHDFS implementation.
+ */
+public class SliderURLConnectionFactory {
+ private static final Logger log =
+ LoggerFactory.getLogger(SliderURLConnectionFactory.class);
+
+ /**
+ * Timeout for socket connects and reads
+ */
+ public final static int DEFAULT_SOCKET_TIMEOUT = 60 * 1000; // 1 minute
+ private final ConnectionConfigurator connConfigurator;
+
+ private static final ConnectionConfigurator DEFAULT_CONFIGURATOR = new BasicConfigurator();
+
+ /**
+ * Construct a new URLConnectionFactory based on the configuration. It will
+ * try to load SSL certificates when it is specified.
+ */
+ public static SliderURLConnectionFactory newInstance(Configuration conf) {
+ ConnectionConfigurator conn;
+ try {
+ conn = newSslConnConfigurator(DEFAULT_SOCKET_TIMEOUT, conf);
+ } catch (Exception e) {
+ log.debug("Cannot load customized SSL configuration.", e);
+ conn = DEFAULT_CONFIGURATOR;
+ }
+ return new SliderURLConnectionFactory(conn);
+ }
+
+ private SliderURLConnectionFactory(ConnectionConfigurator connConfigurator) {
+ this.connConfigurator = connConfigurator;
+ }
+
+ /**
+ * Create a new ConnectionConfigurator for SSL connections
+ */
+ private static ConnectionConfigurator newSslConnConfigurator(final int timeout,
+ Configuration conf) throws IOException, GeneralSecurityException {
+ final SSLFactory factory;
+ final SSLSocketFactory sf;
+ final HostnameVerifier hv;
+
+ factory = new SSLFactory(SSLFactory.Mode.CLIENT, conf);
+ factory.init();
+ sf = factory.createSSLSocketFactory();
+ hv = factory.getHostnameVerifier();
+
+ return new ConnectionConfigurator() {
+ @Override
+ public HttpURLConnection configure(HttpURLConnection conn)
+ throws IOException {
+ if (conn instanceof HttpsURLConnection) {
+ HttpsURLConnection c = (HttpsURLConnection) conn;
+ c.setSSLSocketFactory(sf);
+ c.setHostnameVerifier(hv);
+ }
+ SliderURLConnectionFactory.setupConnection(conn, timeout);
+ return conn;
+ }
+ };
+ }
+
+ /**
+ * Opens a url with read and connect timeouts
+ *
+ * @param url
+ * to open
+ * @return URLConnection
+ * @throws IOException
+ */
+ public URLConnection openConnection(URL url) throws IOException {
+ try {
+ return openConnection(url, false);
+ } catch (AuthenticationException e) {
+ // Unreachable
+ return null;
+ }
+ }
+
+ /**
+ * Opens a url with read and connect timeouts
+ *
+ * @param url
+ * URL to open
+ * @param isSpnego
+ * whether the url should be authenticated via SPNEGO
+ * @return URLConnection
+ * @throws IOException
+ * @throws AuthenticationException
+ */
+ public URLConnection openConnection(URL url, boolean isSpnego)
+ throws IOException, AuthenticationException {
+ if (isSpnego) {
+ log.debug("open AuthenticatedURL connection {}", url);
+ UserGroupInformation.getCurrentUser().checkTGTAndReloginFromKeytab();
+ final AuthenticatedURL.Token authToken = new AuthenticatedURL.Token();
+ return new AuthenticatedURL(new KerberosUgiAuthenticator(),
+ connConfigurator).openConnection(url, authToken);
+ } else {
+ log.debug("open URL connection {}", url);
+ URLConnection connection = url.openConnection();
+ if (connection instanceof HttpURLConnection) {
+ connConfigurator.configure((HttpURLConnection) connection);
+ }
+ return connection;
+ }
+ }
+
+ /**
+ * Sets connection parameters on the given URLConnection
+ *
+ * @param connection
+ * URLConnection to set
+ * @param socketTimeout
+ * the connection and read timeout of the connection.
+ */
+ private static void setupConnection(URLConnection connection, int socketTimeout) {
+ connection.setConnectTimeout(socketTimeout);
+ connection.setReadTimeout(socketTimeout);
+ connection.setUseCaches(false);
+ if (connection instanceof HttpURLConnection) {
+ ((HttpURLConnection) connection).setInstanceFollowRedirects(true);
+ }
+ }
+
+ private static class BasicConfigurator implements ConnectionConfigurator {
+ @Override
+ public HttpURLConnection configure(HttpURLConnection conn)
+ throws IOException {
+ SliderURLConnectionFactory.setupConnection(conn, DEFAULT_SOCKET_TIMEOUT);
+ return conn;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cecc3fc/slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java b/slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java
index 2526684..d19c621 100644
--- a/slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java
+++ b/slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java
@@ -88,8 +88,7 @@ public class UgiJerseyBinding implements
try {
// open a connection handling status codes and so redirections
// but as it opens a connection, it's less useful than you think.
-// return operations.openConnectionRedirecting(url);
-
+
return operations.openConnection(url);
} catch (AuthenticationException e) {
throw new IOException(e);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cecc3fc/slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java b/slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java
index 49fe5ea..e0d2315 100644
--- a/slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java
+++ b/slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java
@@ -22,7 +22,6 @@ import com.google.common.base.Preconditions;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.hdfs.web.URLConnectionFactory;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authentication.client.AuthenticationException;
@@ -31,24 +30,21 @@ import org.apache.hadoop.yarn.webapp.NotFoundException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.servlet.http.HttpServletResponse;
-import javax.ws.rs.core.HttpHeaders;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.HttpURLConnection;
-import java.net.URI;
import java.net.URL;
/**
- * Operations on the JDK UrlConnection class. This uses WebHDFS
- * methods to set up the operations.
+ * Operations on the JDK UrlConnection class.
+ *
*/
public class UrlConnectionOperations extends Configured {
private static final Logger log =
LoggerFactory.getLogger(UrlConnectionOperations.class);
- private URLConnectionFactory connectionFactory;
+ private SliderURLConnectionFactory connectionFactory;
private boolean useSpnego = false;
@@ -59,8 +55,7 @@ public class UrlConnectionOperations extends Configured {
*/
public UrlConnectionOperations(Configuration conf) {
super(conf);
- connectionFactory = URLConnectionFactory
- .newDefaultURLConnectionFactory(conf);
+ connectionFactory = SliderURLConnectionFactory.newInstance(conf);
if (UserGroupInformation.isSecurityEnabled()) {
log.debug("SPNEGO is enabled");
setUseSpnego(true);
@@ -88,18 +83,6 @@ public class UrlConnectionOperations extends Configured {
public HttpURLConnection openConnection(URL url) throws
IOException,
AuthenticationException {
-
- HttpURLConnection conn = innerOpenConnection(url);
- conn.setUseCaches(false);
- conn.setInstanceFollowRedirects(true);
- return conn;
- }
-
-
-
- protected HttpURLConnection innerOpenConnection(URL url) throws
- IOException,
- AuthenticationException {
Preconditions.checkArgument(url.getPort() != 0, "no port");
return (HttpURLConnection) connectionFactory.openConnection(url, useSpnego);
}
[3/6] incubator-slider git commit: SLIDER-878 turn off method
workaround
Posted by st...@apache.org.
SLIDER-878 turn off method workaround
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/13ec0af5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/13ec0af5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/13ec0af5
Branch: refs/heads/develop
Commit: 13ec0af54944fb2327d9123a57b3d3076548a689
Parents: 1cecc3f
Author: Steve Loughran <st...@apache.org>
Authored: Mon May 18 20:41:42 2015 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Mon May 18 20:41:42 2015 +0100
----------------------------------------------------------------------
.../java/org/apache/slider/core/registry/retrieve/AMWebClient.java | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/13ec0af5/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/AMWebClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/AMWebClient.java b/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/AMWebClient.java
index dd9af16..605b46c 100644
--- a/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/AMWebClient.java
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/AMWebClient.java
@@ -55,9 +55,11 @@ public class AMWebClient {
clientConfig.getFeatures().put(
JSONConfiguration.FEATURE_POJO_MAPPING,
Boolean.TRUE);
+/*
clientConfig.getProperties().put(
URLConnectionClientHandler.PROPERTY_HTTP_URL_CONNECTION_SET_METHOD_WORKAROUND,
true);
+*/
URLConnectionClientHandler handler = getUrlConnectionClientHandler();
client = new Client(handler, clientConfig);
client.setFollowRedirects(true);
[4/6] incubator-slider git commit: SLIDER-878 re-enable Jersey method
workaround option in AMClient
Posted by st...@apache.org.
SLIDER-878 re-enable Jersey method workaround option in AMClient
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f1efe763
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f1efe763
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f1efe763
Branch: refs/heads/develop
Commit: f1efe763eb426398669e00ff2a2b4a84651adfd2
Parents: 13ec0af
Author: Steve Loughran <st...@apache.org>
Authored: Tue May 19 10:40:38 2015 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue May 19 10:40:38 2015 +0100
----------------------------------------------------------------------
.../java/org/apache/slider/core/registry/retrieve/AMWebClient.java | 2 --
1 file changed, 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f1efe763/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/AMWebClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/AMWebClient.java b/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/AMWebClient.java
index 605b46c..dd9af16 100644
--- a/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/AMWebClient.java
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/AMWebClient.java
@@ -55,11 +55,9 @@ public class AMWebClient {
clientConfig.getFeatures().put(
JSONConfiguration.FEATURE_POJO_MAPPING,
Boolean.TRUE);
-/*
clientConfig.getProperties().put(
URLConnectionClientHandler.PROPERTY_HTTP_URL_CONNECTION_SET_METHOD_WORKAROUND,
true);
-*/
URLConnectionClientHandler handler = getUrlConnectionClientHandler();
client = new Client(handler, clientConfig);
client.setFollowRedirects(true);