You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2017/06/26 12:50:23 UTC
[49/50] hadoop git commit: HADOOP-14461 Azure: handle failure
gracefully in case of missing account access key. Contributed by Mingliang
Liu.
HADOOP-14461 Azure: handle failure gracefully in case of missing account access key.
Contributed by Mingliang Liu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/48f4a229
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/48f4a229
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/48f4a229
Branch: refs/heads/HADOOP-13345
Commit: 48f4a229a93b65df9e99df2505719dd3538f1188
Parents: 379f19a
Author: Steve Loughran <st...@apache.org>
Authored: Mon Jun 26 13:39:47 2017 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Jun 26 13:39:47 2017 +0100
----------------------------------------------------------------------
.../fs/azure/AzureNativeFileSystemStore.java | 42 ++++++++++----------
.../fs/azure/AzureBlobStorageTestAccount.java | 37 ++++++++++-------
.../TestAzureFileSystemErrorConditions.java | 13 +++---
...TestFileSystemOperationExceptionMessage.java | 13 +++---
4 files changed, 59 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/48f4a229/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
index f76b44a..3fa1a62 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
@@ -303,6 +303,14 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
private boolean useLocalSasKeyMode = false;
private String delegationToken;
+
+ /** The error message template when container is not accessible. */
+ static final String NO_ACCESS_TO_CONTAINER_MSG = "No credentials found for "
+ + "account %s in the configuration, and its container %s is not "
+ + "accessible using anonymous credentials. Please check if the container "
+ + "exists first. If it is not publicly available, you have to provide "
+ + "account credentials.";
+
/**
* A test hook interface that can modify the operation context we use for
* Azure Storage operations, e.g. to inject errors.
@@ -778,18 +786,17 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
rootDirectory = container.getDirectoryReference("");
// Check for container existence, and our ability to access it.
+ boolean canAccess;
try {
- if (!container.exists(getInstrumentedContext())) {
- throw new AzureException("Container " + containerName + " in account "
- + accountName + " not found, and we can't create"
- + " it using anoynomous credentials, and no credentials found for them"
- + " in the configuration.");
- }
+ canAccess = container.exists(getInstrumentedContext());
} catch (StorageException ex) {
- throw new AzureException("Unable to access container " + containerName
- + " in account " + accountName
- + " using anonymous credentials, and no credentials found for them "
- + " in the configuration.", ex);
+ LOG.error("Service returned StorageException when checking existence "
+ + "of container {} in account {}", containerName, accountName, ex);
+ canAccess = false;
+ }
+ if (!canAccess) {
+ throw new AzureException(String.format(NO_ACCESS_TO_CONTAINER_MSG,
+ accountName, containerName));
}
// Accessing the storage server unauthenticated using
@@ -999,22 +1006,17 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
// Check whether the account is configured with an account key.
propertyValue = getAccountKeyFromConfiguration(accountName,
sessionConfiguration);
- if (propertyValue != null) {
-
+ if (StringUtils.isNotEmpty(propertyValue)) {
// Account key was found.
// Create the Azure storage session using the account key and container.
connectUsingConnectionStringCredentials(
getAccountFromAuthority(sessionUri),
getContainerFromAuthority(sessionUri), propertyValue);
-
- // Return to caller
- return;
+ } else {
+ LOG.debug("The account access key is not configured for {}. "
+ + "Now try anonymous access.", sessionUri);
+ connectUsingAnonymousCredentials(sessionUri);
}
-
- // The account access is not configured for this cluster. Try anonymous
- // access.
- connectUsingAnonymousCredentials(sessionUri);
-
} catch (Exception e) {
// Caught exception while attempting to initialize the Azure File
// System store, re-throw the exception.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/48f4a229/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java
index 5f66fd2f..b6c252f 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java
@@ -22,6 +22,9 @@ import com.microsoft.azure.storage.*;
import com.microsoft.azure.storage.blob.*;
import com.microsoft.azure.storage.core.Base64;
import org.apache.commons.configuration2.SubsetConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation;
@@ -46,6 +49,8 @@ import static org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.KEY_USE_LOCA
* for instructions on how to connect to a real Azure Storage account.
*/
public final class AzureBlobStorageTestAccount {
+ private static final Logger LOG = LoggerFactory.getLogger(
+ AzureBlobStorageTestAccount.class);
private static final String ACCOUNT_KEY_PROPERTY_NAME = "fs.azure.account.key.";
private static final String SAS_PROPERTY_NAME = "fs.azure.sas.";
@@ -299,10 +304,9 @@ public final class AzureBlobStorageTestAccount {
Configuration conf = createTestConfiguration();
if (!conf.getBoolean(USE_EMULATOR_PROPERTY_NAME, false)) {
// Not configured to test against the storage emulator.
- System.out
- .println("Skipping emulator Azure test because configuration " +
- "doesn't indicate that it's running." +
- " Please see RunningLiveWasbTests.txt for guidance.");
+ LOG.warn("Skipping emulator Azure test because configuration doesn't "
+ + "indicate that it's running. Please see RunningLiveWasbTests.txt "
+ + "for guidance.");
return null;
}
CloudStorageAccount account =
@@ -456,18 +460,22 @@ public final class AzureBlobStorageTestAccount {
KeyProviderException {
String accountKey = AzureNativeFileSystemStore
.getAccountKeyFromConfiguration(accountName, conf);
- StorageCredentials credentials;
- if (accountKey == null && allowAnonymous) {
- credentials = StorageCredentialsAnonymous.ANONYMOUS;
+ final StorageCredentials credentials;
+ if (accountKey == null) {
+ if (allowAnonymous) {
+ credentials = StorageCredentialsAnonymous.ANONYMOUS;
+ } else {
+ LOG.warn("Skipping live Azure test because of missing key for"
+ + " account '" + accountName + "'. "
+ + "Please see RunningLiveWasbTests.txt for guidance.");
+ return null;
+ }
} else {
credentials = new StorageCredentialsAccountAndKey(
accountName.split("\\.")[0], accountKey);
}
- if (credentials == null) {
- return null;
- } else {
- return new CloudStorageAccount(credentials);
- }
+
+ return new CloudStorageAccount(credentials);
}
public static Configuration createTestConfiguration() {
@@ -493,9 +501,8 @@ public final class AzureBlobStorageTestAccount {
throws URISyntaxException, KeyProviderException {
String testAccountName = conf.get(TEST_ACCOUNT_NAME_PROPERTY_NAME);
if (testAccountName == null) {
- System.out
- .println("Skipping live Azure test because of missing test account." +
- " Please see RunningLiveWasbTests.txt for guidance.");
+ LOG.warn("Skipping live Azure test because of missing test account. "
+ + "Please see RunningLiveWasbTests.txt for guidance.");
return null;
}
return createStorageAccount(testAccountName, conf, false);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/48f4a229/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestAzureFileSystemErrorConditions.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestAzureFileSystemErrorConditions.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestAzureFileSystemErrorConditions.java
index 810bcf7..c985224 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestAzureFileSystemErrorConditions.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestAzureFileSystemErrorConditions.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.fs.azure;
+import static org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.NO_ACCESS_TO_CONTAINER_MSG;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@@ -35,6 +36,8 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.TestHookOperationContext;
+import org.apache.hadoop.test.GenericTestUtils;
+
import org.junit.Test;
import com.microsoft.azure.storage.OperationContext;
@@ -64,18 +67,18 @@ public class TestAzureFileSystemErrorConditions {
*/
@Test
public void testAccessUnauthorizedPublicContainer() throws Exception {
+ final String container = "nonExistentContainer";
+ final String account = "hopefullyNonExistentAccount";
Path noAccessPath = new Path(
- "wasb://nonExistentContainer@hopefullyNonExistentAccount/someFile");
+ "wasb://" + container + "@" + account + "/someFile");
NativeAzureFileSystem.suppressRetryPolicy();
try {
FileSystem.get(noAccessPath.toUri(), new Configuration())
.open(noAccessPath);
assertTrue("Should've thrown.", false);
} catch (AzureException ex) {
- assertTrue("Unexpected message in exception " + ex,
- ex.getMessage().contains(
- "Unable to access container nonExistentContainer in account" +
- " hopefullyNonExistentAccount"));
+ GenericTestUtils.assertExceptionContains(
+ String.format(NO_ACCESS_TO_CONTAINER_MSG, account, container), ex);
} finally {
NativeAzureFileSystem.resumeRetryPolicy();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/48f4a229/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationExceptionMessage.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationExceptionMessage.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationExceptionMessage.java
index 57920a4..e619817 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationExceptionMessage.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationExceptionMessage.java
@@ -21,9 +21,13 @@ import java.net.URI;
import java.util.UUID;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.test.GenericTestUtils;
+
import org.junit.Assert;
import org.junit.Test;
+import static org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.NO_ACCESS_TO_CONTAINER_MSG;
+
public class TestFileSystemOperationExceptionMessage extends
NativeAzureFileSystemBaseTest {
@@ -41,11 +45,6 @@ public class TestFileSystemOperationExceptionMessage extends
String wasbUri = String.format("wasb://%s@%s",
testContainer, testStorageAccount);
- String expectedErrorMessage =
- String.format("Container %s in account %s not found, and we can't create it "
- + "using anoynomous credentials, and no credentials found for "
- + "them in the configuration.", testContainer, testStorageAccount);
-
fs = new NativeAzureFileSystem();
try {
fs.initialize(new URI(wasbUri), conf);
@@ -63,7 +62,9 @@ public class TestFileSystemOperationExceptionMessage extends
|| exceptionMessage.length() == 0) {
Assert.fail();}
else {
- Assert.assertTrue(exceptionMessage.equals(expectedErrorMessage));
+ GenericTestUtils.assertExceptionContains(String.format(
+ NO_ACCESS_TO_CONTAINER_MSG, testStorageAccount, testContainer),
+ ex);
}
} else {
Assert.fail();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org