You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ma...@apache.org on 2018/05/09 21:16:20 UTC
hadoop git commit: HADOOP-15356. Make HTTP timeout configurable in
ADLS connector. Contributed by Atul Sikaria and Sean Mackrory.
Repository: hadoop
Updated Branches:
refs/heads/trunk af4fc2e62 -> 1cfe7506f
HADOOP-15356. Make HTTP timeout configurable in ADLS connector. Contributed by Atul Sikaria and Sean Mackrory.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1cfe7506
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1cfe7506
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1cfe7506
Branch: refs/heads/trunk
Commit: 1cfe7506f7e9aff808af4ec0e57639130a6d0f35
Parents: af4fc2e
Author: Sean Mackrory <ma...@apache.org>
Authored: Fri Apr 27 17:17:02 2018 -0600
Committer: Sean Mackrory <ma...@apache.org>
Committed: Wed May 9 15:15:51 2018 -0600
----------------------------------------------------------------------
.../src/main/resources/core-default.xml | 9 +++
.../org/apache/hadoop/fs/adl/AdlConfKeys.java | 1 +
.../org/apache/hadoop/fs/adl/AdlFileSystem.java | 13 ++++
.../src/site/markdown/troubleshooting_adl.md | 9 +++
.../fs/adl/live/AdlStorageConfiguration.java | 12 ++-
.../fs/adl/live/TestAdlSdkConfiguration.java | 77 ++++++++++++++++++++
6 files changed, 117 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1cfe7506/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index dc13cb9..98b0ddf 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -2929,6 +2929,15 @@
</description>
</property>
+ <property>
+ <name>adl.http.timeout</name>
+ <value>-1</value>
+ <description>
+ Base timeout (in seconds) for HTTP requests from the ADL SDK. Values of
+ zero or less cause the SDK default to be used instead.
+ </description>
+ </property>
+
<!-- Azure Data Lake File System Configurations Ends Here-->
<property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1cfe7506/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
index e3a4ad6..e124e11 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
@@ -105,6 +105,7 @@ public final class AdlConfKeys {
static final String ADL_ENABLEUPN_FOR_OWNERGROUP_KEY =
"adl.feature.ownerandgroup.enableupn";
static final boolean ADL_ENABLEUPN_FOR_OWNERGROUP_DEFAULT = false;
+ public static final String ADL_HTTP_TIMEOUT = "adl.http.timeout";
public static void addDeprecatedKeys() {
Configuration.addDeprecations(new DeprecationDelta[]{
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1cfe7506/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
index aa6babe..3e149a6 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
@@ -195,6 +195,14 @@ public class AdlFileSystem extends FileSystem {
VersionInfo.getVersion().trim() + "/" + clusterName + "/"
+ clusterType);
+ int timeout = conf.getInt(ADL_HTTP_TIMEOUT, -1);
+ if (timeout > 0) {
+ // only set timeout if specified in config. Otherwise use SDK default
+ options.setDefaultTimeout(timeout);
+ } else {
+ LOG.info("No valid ADL SDK timeout configured: using SDK default.");
+ }
+
adlClient.setOptions(options);
boolean trackLatency = conf
@@ -320,6 +328,11 @@ public class AdlFileSystem extends FileSystem {
return azureTokenProvider;
}
+ @VisibleForTesting
+ public ADLStoreClient getAdlClient() {
+ return adlClient;
+ }
+
/**
* Constructing home directory locally is fine as long as Hadoop
* local user name and ADL user name relationship story is not fully baked
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1cfe7506/hadoop-tools/hadoop-azure-datalake/src/site/markdown/troubleshooting_adl.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/site/markdown/troubleshooting_adl.md b/hadoop-tools/hadoop-azure-datalake/src/site/markdown/troubleshooting_adl.md
index 80b2a6f..b362a9c 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/site/markdown/troubleshooting_adl.md
+++ b/hadoop-tools/hadoop-azure-datalake/src/site/markdown/troubleshooting_adl.md
@@ -144,3 +144,12 @@ org.apache.hadoop.security.AccessControlException: MKDIRS failed with error 0x83
```
See "Adding the service principal to your ADL Account".
+
+## Timeouts
+
+The timeout used by the ADL SDK can be overridden with the hadoop property
+`adl.http.timeout`. Some timeouts in compute frameworks may need to be
+addressed by lowering the timeout used by the SDK. A lower timeout at the
+storage layer may allow more retries to be attempted and actually increase
+the likelihood of success before hitting the framework's timeout, as attempts
+that may ultimately fail will fail faster.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1cfe7506/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/AdlStorageConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/AdlStorageConfiguration.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/AdlStorageConfiguration.java
index a9b4ab8..1da40ad 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/AdlStorageConfiguration.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/AdlStorageConfiguration.java
@@ -79,20 +79,24 @@ public final class AdlStorageConfiguration {
if (conf == null) {
conf = getConfiguration();
}
+ return createStorageConnector(conf);
+ }
+ public synchronized static FileSystem createStorageConnector(
+ Configuration fsConfig) throws URISyntaxException, IOException {
if (!isContractTestEnabled()) {
return null;
}
- String fileSystem = conf.get(FILE_SYSTEM_KEY);
+ String fileSystem = fsConfig.get(FILE_SYSTEM_KEY);
if (fileSystem == null || fileSystem.trim().length() == 0) {
throw new IOException("Default file system not configured.");
}
- Class<?> clazz = conf.getClass(FILE_SYSTEM_IMPL_KEY,
+ Class<?> clazz = fsConfig.getClass(FILE_SYSTEM_IMPL_KEY,
FILE_SYSTEM_IMPL_DEFAULT);
- FileSystem fs = (FileSystem) ReflectionUtils.newInstance(clazz, conf);
- fs.initialize(new URI(fileSystem), conf);
+ FileSystem fs = (FileSystem) ReflectionUtils.newInstance(clazz, fsConfig);
+ fs.initialize(new URI(fileSystem), fsConfig);
return fs;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1cfe7506/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlSdkConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlSdkConfiguration.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlSdkConfiguration.java
new file mode 100644
index 0000000..ca762d9
--- /dev/null
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlSdkConfiguration.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.fs.adl.live;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.adl.AdlFileSystem;
+import org.junit.Assert;
+import org.junit.Assume;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+
+import static org.apache.hadoop.fs.adl.AdlConfKeys.ADL_HTTP_TIMEOUT;
+
+/**
+ * Tests interactions with SDK and ensures configuration is having the desired
+ * effect.
+ */
+public class TestAdlSdkConfiguration {
+ @Test
+ public void testDefaultTimeout() throws IOException {
+ AdlFileSystem fs = null;
+ Configuration conf = null;
+ int effectiveTimeout;
+
+ conf = AdlStorageConfiguration.getConfiguration();
+ conf.setInt(ADL_HTTP_TIMEOUT, -1);
+ try {
+ fs = (AdlFileSystem)
+ (AdlStorageConfiguration.createStorageConnector(conf));
+ } catch (URISyntaxException e) {
+ throw new IllegalStateException("Can not initialize ADL FileSystem. "
+ + "Please check test.fs.adl.name property.", e);
+ }
+
+ // Skip this test if we can't get a real FS
+ Assume.assumeNotNull(fs);
+
+ effectiveTimeout = fs.getAdlClient().getDefaultTimeout();
+ Assert.assertFalse("A negative timeout is not supposed to take effect",
+ effectiveTimeout < 0);
+
+ conf = AdlStorageConfiguration.getConfiguration();
+ conf.setInt(ADL_HTTP_TIMEOUT, 17);
+ try {
+ fs = (AdlFileSystem)
+ (AdlStorageConfiguration.createStorageConnector(conf));
+ } catch (URISyntaxException e) {
+ throw new IllegalStateException("Can not initialize ADL FileSystem. "
+ + "Please check test.fs.adl.name property.", e);
+ }
+
+ effectiveTimeout = fs.getAdlClient().getDefaultTimeout();
+ Assert.assertEquals("Timeout is getting set",
+ effectiveTimeout, 17);
+
+ // The default value may vary by SDK, so that value is not tested here.
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org