You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2021/02/03 14:30:14 UTC
[hadoop] branch trunk updated: HADOOP-17337. S3A NetworkBinding has
a runtime dependency on shaded httpclient. (#2599)
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 26b9d48 HADOOP-17337. S3A NetworkBinding has a runtime dependency on shaded httpclient. (#2599)
26b9d48 is described below
commit 26b9d480e802bb288fa3c304a243809c6f19a963
Author: Steve Loughran <st...@cloudera.com>
AuthorDate: Wed Feb 3 14:29:56 2021 +0000
HADOOP-17337. S3A NetworkBinding has a runtime dependency on shaded httpclient. (#2599)
Contributed by Steve Loughran.
---
.../s3a/impl/ConfigureShadedAWSSocketFactory.java | 47 +++++++++++++++++++
.../apache/hadoop/fs/s3a/impl/NetworkBinding.java | 52 +++++++++++-----------
2 files changed, 74 insertions(+), 25 deletions(-)
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ConfigureShadedAWSSocketFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ConfigureShadedAWSSocketFactory.java
new file mode 100644
index 0000000..8205668
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ConfigureShadedAWSSocketFactory.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.impl;
+
+import javax.net.ssl.HostnameVerifier;
+import java.io.IOException;
+
+import com.amazonaws.ClientConfiguration;
+import com.amazonaws.thirdparty.apache.http.conn.ssl.SSLConnectionSocketFactory;
+
+import org.apache.hadoop.security.ssl.DelegatingSSLSocketFactory;
+
+/**
+ * This interacts with the Shaded httpclient library used in the full
+ * AWS SDK. If the S3A client is used with the unshaded SDK, this
+ * class will not link.
+ */
+public class ConfigureShadedAWSSocketFactory implements
+ NetworkBinding.ConfigureAWSSocketFactory {
+
+ @Override
+ public void configureSocketFactory(final ClientConfiguration awsConf,
+ final DelegatingSSLSocketFactory.SSLChannelMode channelMode)
+ throws IOException {
+ DelegatingSSLSocketFactory.initializeDefaultFactory(channelMode);
+ awsConf.getApacheHttpClientConfig().setSslSocketFactory(
+ new SSLConnectionSocketFactory(
+ DelegatingSSLSocketFactory.getDefaultFactory(),
+ (HostnameVerifier) null));
+ }
+}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/NetworkBinding.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/NetworkBinding.java
index 3eeb01f..409ac7b 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/NetworkBinding.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/NetworkBinding.java
@@ -19,14 +19,10 @@
package org.apache.hadoop.fs.s3a.impl;
import java.io.IOException;
-import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.net.URI;
import java.net.URISyntaxException;
-import javax.net.ssl.HostnameVerifier;
-import javax.net.ssl.SSLSocketFactory;
-
import com.amazonaws.ClientConfiguration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -43,20 +39,21 @@ import static org.apache.hadoop.fs.s3a.Constants.SSL_CHANNEL_MODE;
/**
* Configures network settings when communicating with AWS services.
*/
-public class NetworkBinding {
+public final class NetworkBinding {
private static final Logger LOG =
LoggerFactory.getLogger(NetworkBinding.class);
- private static final String AWS_SOCKET_FACTORY_CLASSNAME = "com.amazonaws" +
- ".thirdparty.apache.http.conn.ssl.SSLConnectionSocketFactory";
+ private static final String BINDING_CLASSNAME = "org.apache.hadoop.fs.s3a.impl.ConfigureShadedAWSSocketFactory";
+
+ private NetworkBinding() {
+ }
/**
* Configures the {@code SSLConnectionSocketFactory} used by the AWS SDK.
* A custom Socket Factory can be set using the method
* {@code setSslSocketFactory()}.
- * If {@code SSLConnectionSocketFactory} cannot be found on the classpath, the value
- * of {@link org.apache.hadoop.fs.s3a.Constants#SSL_CHANNEL_MODE} is ignored.
- *
+ * Uses reflection to do this via {@link ConfigureShadedAWSSocketFactory}
+ * so as to avoid
* @param conf the {@link Configuration} used to get the client specified
* value of {@code SSL_CHANNEL_MODE}
* @param awsConf the {@code ClientConfiguration} to set the
@@ -84,29 +81,34 @@ public class NetworkBinding {
DelegatingSSLSocketFactory.initializeDefaultFactory(channelMode);
try {
- // Look for AWS_SOCKET_FACTORY_CLASSNAME on the classpath and instantiate
- // an instance using the DelegatingSSLSocketFactory as the
- // SSLSocketFactory.
- Class<?> sslConnectionSocketFactory = Class.forName(
- AWS_SOCKET_FACTORY_CLASSNAME);
- Constructor<?> factoryConstructor =
- sslConnectionSocketFactory.getDeclaredConstructor(
- SSLSocketFactory.class, HostnameVerifier.class);
- awsConf.getApacheHttpClientConfig().setSslSocketFactory(
- (com.amazonaws.thirdparty.apache.http.conn.ssl.
- SSLConnectionSocketFactory) factoryConstructor
- .newInstance(DelegatingSSLSocketFactory
- .getDefaultFactory(),
- (HostnameVerifier) null));
+ // use reflection to load in our own binding class.
+ // this is *probably* overkill, but it is how we can be fully confident
+ // that no attempt will be made to load/link to the AWS Shaded SDK except
+ // within this try/catch block
+ Class<? extends ConfigureAWSSocketFactory> clazz =
+ (Class<? extends ConfigureAWSSocketFactory>) Class.forName(BINDING_CLASSNAME);
+ clazz.getConstructor()
+ .newInstance()
+ .configureSocketFactory(awsConf, channelMode);
} catch (ClassNotFoundException | NoSuchMethodException |
IllegalAccessException | InstantiationException |
InvocationTargetException | LinkageError e) {
LOG.debug("Unable to create class {}, value of {} will be ignored",
- AWS_SOCKET_FACTORY_CLASSNAME, SSL_CHANNEL_MODE, e);
+ BINDING_CLASSNAME, SSL_CHANNEL_MODE, e);
}
}
/**
+ * Interface used to bind to the socket factory, allows the code which
+ * works with the shaded AWS libraries to exist in their own class.
+ */
+ interface ConfigureAWSSocketFactory {
+ void configureSocketFactory(ClientConfiguration awsConf,
+ DelegatingSSLSocketFactory.SSLChannelMode channelMode)
+ throws IOException;
+ }
+
+ /**
* Given an S3 bucket region as returned by a bucket location query,
* fix it into a form which can be used by other AWS commands.
* <p>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org