[hadoop] branch branch-3.3.5 updated: HADOOP-18499. S3A to support HTTPS web proxies (#5084)

[me...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

mehakmeet pushed a commit to branch branch-3.3.5
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.3.5 by this push:
     new b5db5fa508e HADOOP-18499. S3A to support HTTPS web proxies (#5084)
b5db5fa508e is described below

commit b5db5fa508edd93846f2a336c24a2ac0b7cbcd5f
Author: Mehakmeet Singh <me...@gmail.com>
AuthorDate: Thu Oct 27 20:18:46 2022 +0530

    HADOOP-18499. S3A to support HTTPS web proxies (#5084)
    
    
    The option "fs.s3a.proxy.ssl.enabled" controls
    whether the s3a connects to a proxy over HTTP (default) or HTTPS.
    Set to "true" to use HTTPS.
    
    Contributed by Mehakmeet Singh
---
 .../java/org/apache/hadoop/fs/s3a/Constants.java   |   2 +
 .../java/org/apache/hadoop/fs/s3a/S3AUtils.java    |   8 +-
 .../org/apache/hadoop/fs/s3a/TestS3AProxy.java     | 101 +++++++++++++++++++++
 3 files changed, 109 insertions(+), 2 deletions(-)

diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
index 6d20e3b085a..93cdb40ca49 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
@@ -212,6 +212,8 @@ public final class Constants {
   public static final String PROXY_PASSWORD = "fs.s3a.proxy.password";
   public static final String PROXY_DOMAIN = "fs.s3a.proxy.domain";
   public static final String PROXY_WORKSTATION = "fs.s3a.proxy.workstation";
+  /** Is the proxy secured(proxyProtocol = HTTPS)? */
+  public static final String PROXY_SECURED = "fs.s3a.proxy.ssl.enabled";
 
   /**
    * Number of times the AWS client library should retry errors before
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
index bca2c618334..1401c181e7e 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
@@ -1338,13 +1338,17 @@ public final class S3AUtils {
         LOG.error(msg);
         throw new IllegalArgumentException(msg);
       }
+      boolean isProxySecured = conf.getBoolean(PROXY_SECURED, false);
       awsConf.setProxyUsername(proxyUsername);
       awsConf.setProxyPassword(proxyPassword);
       awsConf.setProxyDomain(conf.getTrimmed(PROXY_DOMAIN));
       awsConf.setProxyWorkstation(conf.getTrimmed(PROXY_WORKSTATION));
+      awsConf.setProxyProtocol(isProxySecured ? Protocol.HTTPS : Protocol.HTTP);
       if (LOG.isDebugEnabled()) {
-        LOG.debug("Using proxy server {}:{} as user {} with password {} on " +
-                "domain {} as workstation {}", awsConf.getProxyHost(),
+        LOG.debug("Using proxy server {}://{}:{} as user {} with password {} "
+                + "on domain {} as workstation {}",
+            awsConf.getProxyProtocol(),
+            awsConf.getProxyHost(),
             awsConf.getProxyPort(),
             String.valueOf(awsConf.getProxyUsername()),
             awsConf.getProxyPassword(), awsConf.getProxyDomain(),
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AProxy.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AProxy.java
new file mode 100644
index 00000000000..e05ee25adfa
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AProxy.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import java.io.IOException;
+
+import com.amazonaws.ClientConfiguration;
+import com.amazonaws.Protocol;
+import org.assertj.core.api.Assertions;
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.test.AbstractHadoopTestBase;
+
+import static org.apache.hadoop.fs.s3a.Constants.PROXY_HOST;
+import static org.apache.hadoop.fs.s3a.Constants.PROXY_PORT;
+import static org.apache.hadoop.fs.s3a.Constants.PROXY_SECURED;
+import static org.apache.hadoop.fs.s3a.S3AUtils.initProxySupport;
+
+/**
+ * Tests to verify {@link S3AUtils} translates the proxy configurations
+ * are set correctly to Client configurations which are later used to construct
+ * the proxy in AWS SDK.
+ */
+public class TestS3AProxy extends AbstractHadoopTestBase {
+
+  /**
+   * Verify Http proxy protocol.
+   */
+  @Test
+  public void testProxyHttp() throws IOException {
+    Configuration proxyConfigForHttp = createProxyConfig(false);
+    verifyProxy(proxyConfigForHttp, false);
+  }
+
+  /**
+   * Verify Https proxy protocol.
+   */
+  @Test
+  public void testProxyHttps() throws IOException {
+    Configuration proxyConfigForHttps = createProxyConfig(true);
+    verifyProxy(proxyConfigForHttps, true);
+  }
+
+  /**
+   * Verify default proxy protocol.
+   */
+  @Test
+  public void testProxyDefault() throws IOException {
+    Configuration proxyConfigDefault = new Configuration();
+    proxyConfigDefault.set(PROXY_HOST, "testProxyDefault");
+    verifyProxy(proxyConfigDefault, false);
+  }
+
+  /**
+   * Assert that the configuration set for a proxy gets translated to Client
+   * configuration with the correct protocol to be used by AWS SDK.
+   * @param proxyConfig Configuration used to set the proxy configs.
+   * @param isExpectedSecured What is the expected protocol for the proxy to
+   *                          be? true for https, and false for http.
+   * @throws IOException
+   */
+  private void verifyProxy(Configuration proxyConfig,
+      boolean isExpectedSecured)
+      throws IOException {
+    ClientConfiguration awsConf = new ClientConfiguration();
+    initProxySupport(proxyConfig, "test-bucket", awsConf);
+    Assertions.assertThat(awsConf.getProxyProtocol())
+        .describedAs("Proxy protocol not as expected")
+        .isEqualTo(isExpectedSecured ? Protocol.HTTPS : Protocol.HTTP);
+  }
+
+  /**
+   * Create a configuration file with proxy configs.
+   * @param isSecured Should the configured proxy be secured or not?
+   * @return configuration.
+   */
+  private Configuration createProxyConfig(boolean isSecured) {
+    Configuration conf = new Configuration();
+    conf.set(PROXY_HOST, "testProxy");
+    conf.set(PROXY_PORT, "1234");
+    conf.setBoolean(PROXY_SECURED, isSecured);
+    return conf;
+  }
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org