You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by me...@apache.org on 2022/10/27 14:48:06 UTC
[hadoop] branch branch-3.3 updated: HADOOP-18499. S3A to support HTTPS web proxies (#5083)
This is an automated email from the ASF dual-hosted git repository.
mehakmeet pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new 6accb7809f6 HADOOP-18499. S3A to support HTTPS web proxies (#5083)
6accb7809f6 is described below
commit 6accb7809f6a2a556cea05f25c21691f80e63161
Author: Mehakmeet Singh <me...@gmail.com>
AuthorDate: Thu Oct 27 20:17:57 2022 +0530
HADOOP-18499. S3A to support HTTPS web proxies (#5083)
The option "fs.s3a.proxy.ssl.enabled" controls
whether the s3a connects to a proxy over HTTP (default) or HTTPS.
Set to "true" to use HTTPS.
Contributed by Mehakmeet Singh
---
.../java/org/apache/hadoop/fs/s3a/Constants.java | 2 +
.../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 8 +-
.../org/apache/hadoop/fs/s3a/TestS3AProxy.java | 101 +++++++++++++++++++++
3 files changed, 109 insertions(+), 2 deletions(-)
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
index 6d20e3b085a..93cdb40ca49 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
@@ -212,6 +212,8 @@ public final class Constants {
public static final String PROXY_PASSWORD = "fs.s3a.proxy.password";
public static final String PROXY_DOMAIN = "fs.s3a.proxy.domain";
public static final String PROXY_WORKSTATION = "fs.s3a.proxy.workstation";
+ /** Is the proxy secured(proxyProtocol = HTTPS)? */
+ public static final String PROXY_SECURED = "fs.s3a.proxy.ssl.enabled";
/**
* Number of times the AWS client library should retry errors before
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
index bca2c618334..1401c181e7e 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
@@ -1338,13 +1338,17 @@ public final class S3AUtils {
LOG.error(msg);
throw new IllegalArgumentException(msg);
}
+ boolean isProxySecured = conf.getBoolean(PROXY_SECURED, false);
awsConf.setProxyUsername(proxyUsername);
awsConf.setProxyPassword(proxyPassword);
awsConf.setProxyDomain(conf.getTrimmed(PROXY_DOMAIN));
awsConf.setProxyWorkstation(conf.getTrimmed(PROXY_WORKSTATION));
+ awsConf.setProxyProtocol(isProxySecured ? Protocol.HTTPS : Protocol.HTTP);
if (LOG.isDebugEnabled()) {
- LOG.debug("Using proxy server {}:{} as user {} with password {} on " +
- "domain {} as workstation {}", awsConf.getProxyHost(),
+ LOG.debug("Using proxy server {}://{}:{} as user {} with password {} "
+ + "on domain {} as workstation {}",
+ awsConf.getProxyProtocol(),
+ awsConf.getProxyHost(),
awsConf.getProxyPort(),
String.valueOf(awsConf.getProxyUsername()),
awsConf.getProxyPassword(), awsConf.getProxyDomain(),
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AProxy.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AProxy.java
new file mode 100644
index 00000000000..e05ee25adfa
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AProxy.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import java.io.IOException;
+
+import com.amazonaws.ClientConfiguration;
+import com.amazonaws.Protocol;
+import org.assertj.core.api.Assertions;
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.test.AbstractHadoopTestBase;
+
+import static org.apache.hadoop.fs.s3a.Constants.PROXY_HOST;
+import static org.apache.hadoop.fs.s3a.Constants.PROXY_PORT;
+import static org.apache.hadoop.fs.s3a.Constants.PROXY_SECURED;
+import static org.apache.hadoop.fs.s3a.S3AUtils.initProxySupport;
+
+/**
+ * Tests to verify {@link S3AUtils} translates the proxy configurations
+ * are set correctly to Client configurations which are later used to construct
+ * the proxy in AWS SDK.
+ */
+public class TestS3AProxy extends AbstractHadoopTestBase {
+
+ /**
+ * Verify Http proxy protocol.
+ */
+ @Test
+ public void testProxyHttp() throws IOException {
+ Configuration proxyConfigForHttp = createProxyConfig(false);
+ verifyProxy(proxyConfigForHttp, false);
+ }
+
+ /**
+ * Verify Https proxy protocol.
+ */
+ @Test
+ public void testProxyHttps() throws IOException {
+ Configuration proxyConfigForHttps = createProxyConfig(true);
+ verifyProxy(proxyConfigForHttps, true);
+ }
+
+ /**
+ * Verify default proxy protocol.
+ */
+ @Test
+ public void testProxyDefault() throws IOException {
+ Configuration proxyConfigDefault = new Configuration();
+ proxyConfigDefault.set(PROXY_HOST, "testProxyDefault");
+ verifyProxy(proxyConfigDefault, false);
+ }
+
+ /**
+ * Assert that the configuration set for a proxy gets translated to Client
+ * configuration with the correct protocol to be used by AWS SDK.
+ * @param proxyConfig Configuration used to set the proxy configs.
+ * @param isExpectedSecured What is the expected protocol for the proxy to
+ * be? true for https, and false for http.
+ * @throws IOException
+ */
+ private void verifyProxy(Configuration proxyConfig,
+ boolean isExpectedSecured)
+ throws IOException {
+ ClientConfiguration awsConf = new ClientConfiguration();
+ initProxySupport(proxyConfig, "test-bucket", awsConf);
+ Assertions.assertThat(awsConf.getProxyProtocol())
+ .describedAs("Proxy protocol not as expected")
+ .isEqualTo(isExpectedSecured ? Protocol.HTTPS : Protocol.HTTP);
+ }
+
+ /**
+ * Create a configuration file with proxy configs.
+ * @param isSecured Should the configured proxy be secured or not?
+ * @return configuration.
+ */
+ private Configuration createProxyConfig(boolean isSecured) {
+ Configuration conf = new Configuration();
+ conf.set(PROXY_HOST, "testProxy");
+ conf.set(PROXY_PORT, "1234");
+ conf.setBoolean(PROXY_SECURED, isSecured);
+ return conf;
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org