You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2021/11/05 14:22:38 UTC
[hadoop] branch trunk updated: Revert "HADOOP-17873. ABFS: Fix
transient failures in ITestAbfsStreamStatistics and
ITestAbfsRestOperationException (#3341)"
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 45f164a Revert "HADOOP-17873. ABFS: Fix transient failures in ITestAbfsStreamStatistics and ITestAbfsRestOperationException (#3341)"
45f164a is described below
commit 45f164a85423b2218c545ad58512e4da8b3ce5c0
Author: Steve Loughran <st...@cloudera.com>
AuthorDate: Fri Nov 5 14:21:15 2021 +0000
Revert "HADOOP-17873. ABFS: Fix transient failures in ITestAbfsStreamStatistics and ITestAbfsRestOperationException (#3341)"
This reverts commit 82658a22d6e443fcf443be01b36763cdbab96ce9.
---
hadoop-tools/hadoop-azure/pom.xml | 2 --
.../oauth2/CustomTokenProviderAdapter.java | 6 -----
.../hadoop/fs/azurebfs/services/AbfsClient.java | 5 ----
.../fs/azurebfs/AbstractAbfsIntegrationTest.java | 5 ----
.../azurebfs/ITestAbfsRestOperationException.java | 13 ++++------
.../fs/azurebfs/oauth2/RetryTestTokenProvider.java | 29 ++++++----------------
.../fs/azurebfs/services/TestAbfsClient.java | 4 ---
7 files changed, 13 insertions(+), 51 deletions(-)
diff --git a/hadoop-tools/hadoop-azure/pom.xml b/hadoop-tools/hadoop-azure/pom.xml
index 075716e..1896e15 100644
--- a/hadoop-tools/hadoop-azure/pom.xml
+++ b/hadoop-tools/hadoop-azure/pom.xml
@@ -555,7 +555,6 @@
<exclude>**/azurebfs/ITestAzureBlobFileSystemListStatus.java</exclude>
<exclude>**/azurebfs/extensions/ITestAbfsDelegationTokens.java</exclude>
<exclude>**/azurebfs/ITestSmallWriteOptimization.java</exclude>
- <exclude>**/azurebfs/ITestAbfsStreamStatistics*.java</exclude>
<exclude>**/azurebfs/services/ITestReadBufferManager.java</exclude>
</excludes>
@@ -598,7 +597,6 @@
<include>**/azurebfs/extensions/ITestAbfsDelegationTokens.java</include>
<include>**/azurebfs/ITestSmallWriteOptimization.java</include>
<include>**/azurebfs/services/ITestReadBufferManager.java</include>
- <include>**/azurebfs/ITestAbfsStreamStatistics*.java</include>
</includes>
</configuration>
</execution>
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/CustomTokenProviderAdapter.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/CustomTokenProviderAdapter.java
index 6ae0962..1491410 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/CustomTokenProviderAdapter.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/CustomTokenProviderAdapter.java
@@ -22,7 +22,6 @@ package org.apache.hadoop.fs.azurebfs.oauth2;
import java.io.IOException;
import java.net.URI;
-import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -139,9 +138,4 @@ public final class CustomTokenProviderAdapter extends AccessTokenProvider
String suffix = ExtensionHelper.getUserAgentSuffix(adaptee, "");
return suffix != null ? suffix : "";
}
-
- @VisibleForTesting
- protected CustomTokenProviderAdaptee getCustomTokenProviderAdaptee() {
- return adaptee;
- }
}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java
index 643cb66..17d5c03 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java
@@ -1223,9 +1223,4 @@ public class AbfsClient implements Closeable {
public <V> void addCallback(ListenableFuture<V> future, FutureCallback<V> callback) {
Futures.addCallback(future, callback, executorService);
}
-
- @VisibleForTesting
- protected AccessTokenProvider getTokenProvider() {
- return tokenProvider;
- }
}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java
index e2c274c..56d5538 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java
@@ -37,12 +37,10 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.azurebfs.constants.FSOperationType;
import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException;
-import org.apache.hadoop.fs.azurebfs.oauth2.AccessTokenProvider;
import org.apache.hadoop.fs.azurebfs.security.AbfsDelegationTokenManager;
import org.apache.hadoop.fs.azurebfs.services.AbfsClient;
import org.apache.hadoop.fs.azurebfs.services.AbfsOutputStream;
import org.apache.hadoop.fs.azurebfs.services.AuthType;
-import org.apache.hadoop.fs.azurebfs.services.TestAbfsClient;
import org.apache.hadoop.fs.azure.AzureNativeFileSystemStore;
import org.apache.hadoop.fs.azure.NativeAzureFileSystem;
import org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation;
@@ -243,9 +241,6 @@ public abstract class AbstractAbfsIntegrationTest extends
}
}
- public AccessTokenProvider getAccessTokenProvider(final AzureBlobFileSystem fs) {
- return TestAbfsClient.getAccessTokenProvider(fs.getAbfsStore().getClient());
- }
public void loadConfiguredFileSystem() throws Exception {
// disable auto-creation of filesystem
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java
index b1705ca..a71e7bc 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java
@@ -111,10 +111,7 @@ public class ITestAbfsRestOperationException extends AbstractAbfsIntegrationTest
final AzureBlobFileSystem fs1 =
(AzureBlobFileSystem) FileSystem.newInstance(fs.getUri(),
config);
- RetryTestTokenProvider retryTestTokenProvider
- = RetryTestTokenProvider.getCurrentRetryTestProviderInstance(
- getAccessTokenProvider(fs1));
- retryTestTokenProvider.resetStatusToFirstTokenFetch();
+ RetryTestTokenProvider.ResetStatusToFirstTokenFetch();
intercept(Exception.class,
()-> {
@@ -122,10 +119,10 @@ public class ITestAbfsRestOperationException extends AbstractAbfsIntegrationTest
});
// Number of retries done should be as configured
- Assert.assertEquals(
- "Number of token fetch retries done does not match with fs.azure"
- + ".custom.token.fetch.retry.count configured", numOfRetries,
- retryTestTokenProvider.getRetryCount());
+ Assert.assertTrue(
+ "Number of token fetch retries (" + RetryTestTokenProvider.reTryCount
+ + ") done, does not match with fs.azure.custom.token.fetch.retry.count configured (" + numOfRetries
+ + ")", RetryTestTokenProvider.reTryCount == numOfRetries);
}
@Test
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/oauth2/RetryTestTokenProvider.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/oauth2/RetryTestTokenProvider.java
index 7427add..3566ebb 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/oauth2/RetryTestTokenProvider.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/oauth2/RetryTestTokenProvider.java
@@ -30,12 +30,12 @@ import org.slf4j.LoggerFactory;
*/
public class RetryTestTokenProvider implements CustomTokenProviderAdaptee {
- private static final Logger LOG = LoggerFactory.getLogger(
- RetryTestTokenProvider.class);
-
// Need to track first token fetch otherwise will get counted as a retry too.
- private boolean isThisFirstTokenFetch = true;
- private int retryCount = 0;
+ private static boolean isThisFirstTokenFetch = true;
+ public static int reTryCount = 0;
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(RetryTestTokenProvider.class);
@Override
public void initialize(Configuration configuration, String accountName)
@@ -43,13 +43,9 @@ public class RetryTestTokenProvider implements CustomTokenProviderAdaptee {
}
- /**
- * Clear earlier retry details and reset RetryTestTokenProvider instance to
- * state of first access token fetch call.
- */
- public void resetStatusToFirstTokenFetch() {
+ public static void ResetStatusToFirstTokenFetch() {
isThisFirstTokenFetch = true;
- retryCount = 0;
+ reTryCount = 0;
}
@Override
@@ -57,7 +53,7 @@ public class RetryTestTokenProvider implements CustomTokenProviderAdaptee {
if (isThisFirstTokenFetch) {
isThisFirstTokenFetch = false;
} else {
- retryCount++;
+ reTryCount++;
}
LOG.debug("RetryTestTokenProvider: Throw an exception in fetching tokens");
@@ -68,13 +64,4 @@ public class RetryTestTokenProvider implements CustomTokenProviderAdaptee {
public Date getExpiryTime() {
return new Date();
}
-
- public static RetryTestTokenProvider getCurrentRetryTestProviderInstance(
- AccessTokenProvider customTokenProvider) {
- return (RetryTestTokenProvider) ((CustomTokenProviderAdapter) customTokenProvider).getCustomTokenProviderAdaptee();
- }
-
- public int getRetryCount() {
- return retryCount;
- }
}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java
index 0a1dca7..a725bf3 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java
@@ -395,8 +395,4 @@ public final class TestAbfsClient {
url,
requestHeaders);
}
-
- public static AccessTokenProvider getAccessTokenProvider(AbfsClient client) {
- return client.getTokenProvider();
- }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org