You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2017/06/26 12:50:19 UTC
[45/50] hadoop git commit: HADOOP-14547. [WASB] the configured retry
policy is not used for all storage operations. Contributed by Thomas.
HADOOP-14547. [WASB] the configured retry policy is not used for all storage operations.
Contributed by Thomas.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c6bd73c6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c6bd73c6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c6bd73c6
Branch: refs/heads/HADOOP-13345
Commit: c6bd73c6c5760c3a52203e6a51628587ceec6896
Parents: 0b77262
Author: Steve Loughran <st...@apache.org>
Authored: Fri Jun 23 21:38:41 2017 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Jun 23 21:38:41 2017 +0100
----------------------------------------------------------------------
.../fs/azure/AzureNativeFileSystemStore.java | 11 ++-----
.../fs/azure/SecureStorageInterfaceImpl.java | 30 +++++++++++++++-----
.../hadoop/fs/azure/StorageInterfaceImpl.java | 28 ++++++++++++++----
3 files changed, 49 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c6bd73c6/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
index 534919e..f76b44a 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
@@ -481,6 +481,9 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
}
}
+ // Configure Azure storage session.
+ configureAzureStorageSession();
+
// Start an Azure storage session.
//
createAzureStorageSession();
@@ -792,9 +795,6 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
// Accessing the storage server unauthenticated using
// anonymous credentials.
isAnonymousCredentials = true;
-
- // Configure Azure storage session.
- configureAzureStorageSession();
}
private void connectUsingCredentials(String accountName,
@@ -820,9 +820,6 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
// Can only create container if using account key credentials
canCreateOrModifyContainer = credentials instanceof StorageCredentialsAccountAndKey;
-
- // Configure Azure storage session.
- configureAzureStorageSession();
}
/**
@@ -848,8 +845,6 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
rootDirectory = container.getDirectoryReference("");
canCreateOrModifyContainer = true;
-
- configureAzureStorageSession();
tolerateOobAppends = false;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c6bd73c6/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureStorageInterfaceImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureStorageInterfaceImpl.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureStorageInterfaceImpl.java
index 650149a..810aacf 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureStorageInterfaceImpl.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureStorageInterfaceImpl.java
@@ -69,6 +69,8 @@ public class SecureStorageInterfaceImpl extends StorageInterface {
public static final String SAS_ERROR_CODE = "SAS Error";
private SASKeyGeneratorInterface sasKeyGenerator;
private String storageAccount;
+ private RetryPolicyFactory retryPolicy;
+ private int timeoutIntervalInMs;
public SecureStorageInterfaceImpl(boolean useLocalSASKeyMode,
Configuration conf) throws SecureModeException {
@@ -90,10 +92,12 @@ public class SecureStorageInterfaceImpl extends StorageInterface {
@Override
public void setTimeoutInMs(int timeoutInMs) {
+ timeoutIntervalInMs = timeoutInMs;
}
@Override
public void setRetryPolicyFactory(RetryPolicyFactory retryPolicyFactory) {
+ retryPolicy = retryPolicyFactory;
}
@Override
@@ -133,9 +137,15 @@ public class SecureStorageInterfaceImpl extends StorageInterface {
throws URISyntaxException, StorageException {
try {
- return new SASCloudBlobContainerWrapperImpl(storageAccount,
- new CloudBlobContainer(sasKeyGenerator.getContainerSASUri(
- storageAccount, name)), sasKeyGenerator);
+ CloudBlobContainer container = new CloudBlobContainer(sasKeyGenerator.getContainerSASUri(
+ storageAccount, name));
+ if (retryPolicy != null) {
+ container.getServiceClient().getDefaultRequestOptions().setRetryPolicyFactory(retryPolicy);
+ }
+ if (timeoutIntervalInMs > 0) {
+ container.getServiceClient().getDefaultRequestOptions().setTimeoutIntervalInMs(timeoutIntervalInMs);
+ }
+ return new SASCloudBlobContainerWrapperImpl(storageAccount, container, sasKeyGenerator);
} catch (SASKeyGenerationException sasEx) {
String errorMsg = "Encountered SASKeyGeneration exception while "
+ "generating SAS Key for container : " + name
@@ -216,9 +226,12 @@ public class SecureStorageInterfaceImpl extends StorageInterface {
public CloudBlobWrapper getBlockBlobReference(String relativePath)
throws URISyntaxException, StorageException {
try {
+ CloudBlockBlob blob = new CloudBlockBlob(sasKeyGenerator.getRelativeBlobSASUri(
+ storageAccount, getName(), relativePath));
+ blob.getServiceClient().setDefaultRequestOptions(
+ container.getServiceClient().getDefaultRequestOptions());
return new SASCloudBlockBlobWrapperImpl(
- new CloudBlockBlob(sasKeyGenerator.getRelativeBlobSASUri(
- storageAccount, getName(), relativePath)));
+ blob);
} catch (SASKeyGenerationException sasEx) {
String errorMsg = "Encountered SASKeyGeneration exception while "
+ "generating SAS Key for relativePath : " + relativePath
@@ -232,9 +245,12 @@ public class SecureStorageInterfaceImpl extends StorageInterface {
public CloudBlobWrapper getPageBlobReference(String relativePath)
throws URISyntaxException, StorageException {
try {
+ CloudPageBlob blob = new CloudPageBlob(sasKeyGenerator.getRelativeBlobSASUri(
+ storageAccount, getName(), relativePath));
+ blob.getServiceClient().setDefaultRequestOptions(
+ container.getServiceClient().getDefaultRequestOptions());
return new SASCloudPageBlobWrapperImpl(
- new CloudPageBlob(sasKeyGenerator.getRelativeBlobSASUri(
- storageAccount, getName(), relativePath)));
+ blob);
} catch (SASKeyGenerationException sasEx) {
String errorMsg = "Encountered SASKeyGeneration exception while "
+ "generating SAS Key for relativePath : " + relativePath
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c6bd73c6/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java
index 367cd04..491a0d0 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java
@@ -60,32 +60,50 @@ import com.microsoft.azure.storage.blob.PageRange;
@InterfaceAudience.Private
class StorageInterfaceImpl extends StorageInterface {
private CloudBlobClient serviceClient;
+ private RetryPolicyFactory retryPolicyFactory;
+ private int timeoutIntervalInMs;
+
+ private void updateRetryPolicy() {
+ if (serviceClient != null && retryPolicyFactory != null) {
+ serviceClient.getDefaultRequestOptions().setRetryPolicyFactory(retryPolicyFactory);
+ }
+ }
+
+ private void updateTimeoutInMs() {
+ if (serviceClient != null && timeoutIntervalInMs > 0) {
+ serviceClient.getDefaultRequestOptions().setTimeoutIntervalInMs(timeoutIntervalInMs);
+ }
+ }
@Override
public void setRetryPolicyFactory(final RetryPolicyFactory retryPolicyFactory) {
- serviceClient.getDefaultRequestOptions().setRetryPolicyFactory(
- retryPolicyFactory);
+ this.retryPolicyFactory = retryPolicyFactory;
+ updateRetryPolicy();
}
@Override
public void setTimeoutInMs(int timeoutInMs) {
- serviceClient.getDefaultRequestOptions().setTimeoutIntervalInMs(
- timeoutInMs);
+ timeoutIntervalInMs = timeoutInMs;
+ updateTimeoutInMs();
}
@Override
public void createBlobClient(CloudStorageAccount account) {
serviceClient = account.createCloudBlobClient();
+ updateRetryPolicy();
+ updateTimeoutInMs();
}
@Override
public void createBlobClient(URI baseUri) {
- serviceClient = new CloudBlobClient(baseUri);
+ createBlobClient(baseUri, (StorageCredentials)null);
}
@Override
public void createBlobClient(URI baseUri, StorageCredentials credentials) {
serviceClient = new CloudBlobClient(baseUri, credentials);
+ updateRetryPolicy();
+ updateTimeoutInMs();
}
@Override
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org