You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2019/09/09 16:11:05 UTC
[hadoop] branch trunk updated: HADOOP-16438. ADLS Gen1 OpenSSL
config control.
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 147f986 HADOOP-16438. ADLS Gen1 OpenSSL config control.
147f986 is described below
commit 147f98629cfa799044d5a911221f365a03f9380c
Author: Sneha Vijayarajan <sn...@gmail.com>
AuthorDate: Mon Sep 9 17:09:32 2019 +0100
HADOOP-16438. ADLS Gen1 OpenSSL config control.
Contributed by Sneha Vijayarajan.
Change-Id: Ib79ea6b4a90ad068033e175f3f59c5185868872d
---
.../src/main/resources/core-default.xml | 14 ++++++++++
hadoop-tools/hadoop-azure-datalake/pom.xml | 2 +-
.../java/org/apache/hadoop/fs/adl/AdlConfKeys.java | 1 +
.../org/apache/hadoop/fs/adl/AdlFileSystem.java | 4 +++
.../src/site/markdown/troubleshooting_adl.md | 10 +++++++
.../fs/adl/live/TestAdlSdkConfiguration.java | 32 +++++++++++++++++++++-
6 files changed, 61 insertions(+), 2 deletions(-)
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 7ed3e8a..583f833 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -3373,6 +3373,20 @@
</description>
</property>
+ <property>
+ <name>adl.ssl.channel.mode</name>
+ <value></value>
+ <description>
+ Valid inputs are OpenSSL, Default_JSE and Default (case insensitive).
+ If config is missing or is invalid, SSL Channel mode will be set to Default.
+
+ When OpenSSL, SSL socket connections are created in OpenSSL mode.
+ When Default_JSE, SSL socket connections are created in the default JSE mode.
+ When Default, SSL socket connections are attempted with OpenSSL
+ and will fallback to Default_JSE mode if OpenSSL is not available at runtime.
+ </description>
+ </property>
+
<!-- Azure Data Lake File System Configurations Ends Here-->
<property>
diff --git a/hadoop-tools/hadoop-azure-datalake/pom.xml b/hadoop-tools/hadoop-azure-datalake/pom.xml
index 6e73eaa..9952861 100644
--- a/hadoop-tools/hadoop-azure-datalake/pom.xml
+++ b/hadoop-tools/hadoop-azure-datalake/pom.xml
@@ -33,7 +33,7 @@
<minimalJsonVersion>0.9.1</minimalJsonVersion>
<file.encoding>UTF-8</file.encoding>
<downloadSources>true</downloadSources>
- <azure.data.lake.store.sdk.version>2.3.3</azure.data.lake.store.sdk.version>
+ <azure.data.lake.store.sdk.version>2.3.6</azure.data.lake.store.sdk.version>
</properties>
<build>
<plugins>
diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
index e124e11..5738d46 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
@@ -106,6 +106,7 @@ public final class AdlConfKeys {
"adl.feature.ownerandgroup.enableupn";
static final boolean ADL_ENABLEUPN_FOR_OWNERGROUP_DEFAULT = false;
public static final String ADL_HTTP_TIMEOUT = "adl.http.timeout";
+ public static final String ADL_SSL_CHANNEL_MODE = "adl.ssl.channel.mode";
public static void addDeprecatedKeys() {
Configuration.addDeprecations(new DeprecationDelta[]{
diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
index 79e8a69..3955721 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
@@ -203,6 +203,10 @@ public class AdlFileSystem extends FileSystem {
LOG.info("No valid ADL SDK timeout configured: using SDK default.");
}
+ String sslChannelMode = conf.get(ADL_SSL_CHANNEL_MODE,
+ "Default");
+ options.setSSLChannelMode(sslChannelMode);
+
adlClient.setOptions(options);
boolean trackLatency = conf
diff --git a/hadoop-tools/hadoop-azure-datalake/src/site/markdown/troubleshooting_adl.md b/hadoop-tools/hadoop-azure-datalake/src/site/markdown/troubleshooting_adl.md
index b362a9c..0cc1a18 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/site/markdown/troubleshooting_adl.md
+++ b/hadoop-tools/hadoop-azure-datalake/src/site/markdown/troubleshooting_adl.md
@@ -153,3 +153,13 @@ addressed by lowering the timeout used by the SDK. A lower timeout at the
storage layer may allow more retries to be attempted and actually increase
the likelihood of success before hitting the framework's timeout, as attempts
that may ultimately fail will fail faster.
+
+## SSL Socket Channel Mode
+
+ADL SDK will by default attempt to create secure socket connections over
+OpenSSL as they provide significant performance improvements over Https. If
+there are runtime issues, SDK will default connections over Default_JSE. This
+can be overridden with the hadoop property `adl.ssl.channel.mode`. Possible
+values for this config are OpenSSL, Default_JSE and Default (default).
+Setting the config to OpenSSL or Default_JSE will try the connection to
+only that mode.
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlSdkConfiguration.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlSdkConfiguration.java
index ca762d9..980b683 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlSdkConfiguration.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlSdkConfiguration.java
@@ -19,6 +19,8 @@
package org.apache.hadoop.fs.adl.live;
+import com.microsoft.azure.datalake.store.SSLSocketFactoryEx.SSLChannelMode;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.adl.AdlFileSystem;
import org.junit.Assert;
@@ -29,6 +31,7 @@ import java.io.IOException;
import java.net.URISyntaxException;
import static org.apache.hadoop.fs.adl.AdlConfKeys.ADL_HTTP_TIMEOUT;
+import static org.apache.hadoop.fs.adl.AdlConfKeys.ADL_SSL_CHANNEL_MODE;
/**
* Tests interactions with SDK and ensures configuration is having the desired
@@ -53,7 +56,6 @@ public class TestAdlSdkConfiguration {
// Skip this test if we can't get a real FS
Assume.assumeNotNull(fs);
-
effectiveTimeout = fs.getAdlClient().getDefaultTimeout();
Assert.assertFalse("A negative timeout is not supposed to take effect",
effectiveTimeout < 0);
@@ -74,4 +76,32 @@ public class TestAdlSdkConfiguration {
// The default value may vary by SDK, so that value is not tested here.
}
+
+ @Test
+ public void testSSLChannelModeConfig()
+ throws IOException, URISyntaxException {
+ testSSLChannelMode(SSLChannelMode.OpenSSL, "OpenSSL");
+ testSSLChannelMode(SSLChannelMode.Default_JSE, "Default_JSE");
+ testSSLChannelMode(SSLChannelMode.Default, "Default");
+ // If config set is invalid, SSL channel mode will be Default.
+ testSSLChannelMode(SSLChannelMode.Default, "Invalid");
+ // Config value is case insensitive.
+ testSSLChannelMode(SSLChannelMode.OpenSSL, "openssl");
+ }
+
+ public void testSSLChannelMode(SSLChannelMode expectedMode,
+ String sslChannelModeConfigValue) throws IOException, URISyntaxException {
+
+ AdlFileSystem fs = null;
+ Configuration conf = null;
+
+ conf = AdlStorageConfiguration.getConfiguration();
+ conf.set(ADL_SSL_CHANNEL_MODE, sslChannelModeConfigValue);
+ fs = (AdlFileSystem) (AdlStorageConfiguration.createStorageConnector(conf));
+
+ SSLChannelMode sslChannelMode = fs.getAdlClient().getSSLChannelMode();
+ Assert.assertEquals(
+ "Unexpected SSL Channel Mode for adl.ssl.channel.mode config value : "
+ + sslChannelModeConfigValue, expectedMode, sslChannelMode);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org