You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by da...@apache.org on 2019/12/09 05:34:01 UTC
[hadoop] 06/06: HADOOP-16660. ABFS: Make RetryCount in
ExponentialRetryPolicy Configurable.
This is an automated email from the ASF dual-hosted git repository.
dazhou pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit aa9cd0a2d641bef91b3d9d4054d52e8b0ee61bc1
Author: Sneha Vijayarajan <sn...@gmail.com>
AuthorDate: Wed Nov 27 15:07:27 2019 -0800
HADOOP-16660. ABFS: Make RetryCount in ExponentialRetryPolicy Configurable.
Contributed by Sneha Vijayarajan.
---
.../fs/azurebfs/AzureBlobFileSystemStore.java | 6 +-
.../fs/azurebfs/services/AbfsRestOperation.java | 3 +
.../azurebfs/services/ExponentialRetryPolicy.java | 11 ++--
.../azurebfs/ITestAbfsRestOperationException.java | 41 +++++++++++++
.../TestAbfsConfigurationFieldsValidation.java | 2 +-
.../fs/azurebfs/oauth2/RetryTestTokenProvider.java | 67 ++++++++++++++++++++++
6 files changed, 120 insertions(+), 10 deletions(-)
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
index 27d7f92..3b4c219 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
@@ -1118,7 +1118,9 @@ public class AzureBlobFileSystemStore {
tokenProvider = abfsConfiguration.getTokenProvider();
}
- this.client = new AbfsClient(baseUrl, creds, abfsConfiguration, new ExponentialRetryPolicy(), tokenProvider, abfsPerfTracker);
+ this.client = new AbfsClient(baseUrl, creds, abfsConfiguration,
+ new ExponentialRetryPolicy(abfsConfiguration.getMaxIoRetries()),
+ tokenProvider, abfsPerfTracker);
}
private String getOctalNotation(FsPermission fsPermission) {
@@ -1342,4 +1344,4 @@ public class AzureBlobFileSystemStore {
AbfsClient getClient() {
return this.client;
}
-}
+}
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java
index 4803b6c..ddcb222 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java
@@ -130,8 +130,11 @@ public class AbfsRestOperation {
}
int retryCount = 0;
+ LOG.debug("First execution of REST operation - {}", operationType);
while (!executeHttpOperation(retryCount++)) {
try {
+ LOG.debug("Retrying REST operation {}. RetryCount = {}",
+ operationType, retryCount);
Thread.sleep(client.getRetryPolicy().getRetryInterval(retryCount));
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ExponentialRetryPolicy.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ExponentialRetryPolicy.java
index 5eb7a66..b272cf2 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ExponentialRetryPolicy.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ExponentialRetryPolicy.java
@@ -26,11 +26,6 @@ import java.net.HttpURLConnection;
* */
public class ExponentialRetryPolicy {
/**
- * Represents the default number of retry attempts.
- */
- private static final int DEFAULT_CLIENT_RETRY_COUNT = 30;
-
- /**
* Represents the default amount of time used when calculating a random delta in the exponential
* delay between retries.
*/
@@ -86,8 +81,10 @@ public class ExponentialRetryPolicy {
/**
* Initializes a new instance of the {@link ExponentialRetryPolicy} class.
*/
- public ExponentialRetryPolicy() {
- this(DEFAULT_CLIENT_RETRY_COUNT, DEFAULT_MIN_BACKOFF, DEFAULT_MAX_BACKOFF, DEFAULT_CLIENT_BACKOFF);
+ public ExponentialRetryPolicy(final int maxIoRetries) {
+
+ this(maxIoRetries, DEFAULT_MIN_BACKOFF, DEFAULT_MAX_BACKOFF,
+ DEFAULT_CLIENT_BACKOFF);
}
/**
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java
index ff88b02..6f69826 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java
@@ -20,12 +20,17 @@ package org.apache.hadoop.fs.azurebfs;
import java.io.IOException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.azurebfs.oauth2.RetryTestTokenProvider;
import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.junit.Assert;
import org.junit.Test;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+
/**
* Verify the AbfsRestOperationException error message format.
* */
@@ -72,4 +77,40 @@ public class ITestAbfsRestOperationException extends AbstractAbfsIntegrationTest
&& errorFields[5].contains("Time"));
}
}
+
+ @Test
+ public void testRequestRetryConfig() throws Exception {
+ testRetryLogic(0);
+ testRetryLogic(3);
+ }
+
+ public void testRetryLogic(int numOfRetries) throws Exception {
+ AzureBlobFileSystem fs = this.getFileSystem();
+
+ Configuration config = new Configuration(this.getRawConfiguration());
+ String accountName = config.get("fs.azure.abfs.account.name");
+ // Setup to configure custom token provider
+ config.set("fs.azure.account.auth.type." + accountName, "Custom");
+ config.set("fs.azure.account.oauth.provider.type." + accountName, "org.apache.hadoop.fs"
+ + ".azurebfs.oauth2.RetryTestTokenProvider");
+ config.set("fs.azure.io.retry.max.retries", Integer.toString(numOfRetries));
+ // Stop filesystem creation as it will lead to calls to store.
+ config.set("fs.azure.createRemoteFileSystemDuringInitialization", "false");
+
+ final AzureBlobFileSystem fs1 =
+ (AzureBlobFileSystem) FileSystem.newInstance(fs.getUri(),
+ config);
+ RetryTestTokenProvider.ResetStatusToFirstTokenFetch();
+
+ intercept(Exception.class,
+ ()-> {
+ fs1.getFileStatus(new Path("/"));
+ });
+
+ // Number of retries done should be as configured
+ Assert.assertTrue(
+ "Number of token fetch retries (" + RetryTestTokenProvider.reTryCount
+ + ") done, does not match with max " + "retry count configured (" + numOfRetries
+ + ")", RetryTestTokenProvider.reTryCount == numOfRetries);
+ }
}
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsConfigurationFieldsValidation.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsConfigurationFieldsValidation.java
index 9848228..5c601e2 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsConfigurationFieldsValidation.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsConfigurationFieldsValidation.java
@@ -182,4 +182,4 @@ public class TestAbfsConfigurationFieldsValidation {
assertEquals(SSLSocketFactoryEx.SSLChannelMode.OpenSSL, localAbfsConfiguration.getPreferredSSLFactoryOption());
}
-}
+}
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/oauth2/RetryTestTokenProvider.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/oauth2/RetryTestTokenProvider.java
new file mode 100644
index 0000000..3566ebb
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/oauth2/RetryTestTokenProvider.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.azurebfs.oauth2;
+
+import java.io.IOException;
+import java.util.Date;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.azurebfs.extensions.CustomTokenProviderAdaptee;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Test Token provider which should throw exception and trigger retries
+ */
+public class RetryTestTokenProvider implements CustomTokenProviderAdaptee {
+
+ // Need to track first token fetch otherwise will get counted as a retry too.
+ private static boolean isThisFirstTokenFetch = true;
+ public static int reTryCount = 0;
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(RetryTestTokenProvider.class);
+
+ @Override
+ public void initialize(Configuration configuration, String accountName)
+ throws IOException {
+
+ }
+
+ public static void ResetStatusToFirstTokenFetch() {
+ isThisFirstTokenFetch = true;
+ reTryCount = 0;
+ }
+
+ @Override
+ public String getAccessToken() throws IOException {
+ if (isThisFirstTokenFetch) {
+ isThisFirstTokenFetch = false;
+ } else {
+ reTryCount++;
+ }
+
+ LOG.debug("RetryTestTokenProvider: Throw an exception in fetching tokens");
+ throw new IOException("test exception");
+ }
+
+ @Override
+ public Date getExpiryTime() {
+ return new Date();
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org