You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by tm...@apache.org on 2018/08/31 03:26:17 UTC
hadoop git commit: HADOOP-15663. ABFS: Simplify configuration.
Contributed by Da Zhou.
Repository: hadoop
Updated Branches:
refs/heads/HADOOP-15407 7aabb611d -> d58ea7c96
HADOOP-15663. ABFS: Simplify configuration.
Contributed by Da Zhou.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d58ea7c9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d58ea7c9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d58ea7c9
Branch: refs/heads/HADOOP-15407
Commit: d58ea7c96abe1dbba2f89019676a2dd869e6b550
Parents: 7aabb61
Author: Thomas Marquardt <tm...@microsoft.com>
Authored: Fri Aug 31 03:24:42 2018 +0000
Committer: Thomas Marquardt <tm...@microsoft.com>
Committed: Fri Aug 31 03:24:42 2018 +0000
----------------------------------------------------------------------
.../src/main/resources/core-default.xml | 12 ++
.../hadoop/fs/azurebfs/AbfsConfiguration.java | 4 -
.../fs/azurebfs/AzureBlobFileSystemStore.java | 24 ++-
.../azurebfs/constants/ConfigurationKeys.java | 5 +-
.../hadoop/fs/azurebfs/utils/UriUtils.java | 15 +-
.../src/site/markdown/testing_azure.md | 209 ++++++++++++++-----
.../fs/azure/AzureBlobStorageTestAccount.java | 22 +-
...TestFileSystemOperationExceptionMessage.java | 3 +-
.../fs/azure/ITestWasbUriAndConfiguration.java | 26 ---
.../azure/integration/AzureTestConstants.java | 6 +-
.../fs/azure/integration/AzureTestUtils.java | 18 ++
.../azure/metrics/TestRollingWindowAverage.java | 4 +-
.../azurebfs/AbstractAbfsIntegrationTest.java | 73 ++++---
.../azurebfs/AbstractAbfsTestWithTimeout.java | 70 +++++++
.../ITestAzureBlobFileSystemBackCompat.java | 2 +-
.../fs/azurebfs/ITestWasbAbfsCompatibility.java | 2 +-
.../TestAbfsConfigurationFieldsValidation.java | 3 +-
.../constants/TestConfigurationKeys.java | 13 +-
.../contract/ABFSContractTestBinding.java | 3 +
.../TestConfigurationValidators.java | 1 -
.../hadoop/fs/azurebfs/utils/AbfsTestUtils.java | 80 +++++++
.../utils/CleanUpAbfsTestContainer.java | 77 -------
.../hadoop/fs/azurebfs/utils/TestUriUtils.java | 12 +-
.../src/test/resources/azure-bfs-test.xml | 188 -----------------
.../src/test/resources/azure-test.xml | 56 ++---
25 files changed, 473 insertions(+), 455 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d58ea7c9/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 93649e7..b28f5ec 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -1595,6 +1595,18 @@
<!-- Azure file system properties -->
<property>
+ <name>fs.AbstractFileSystem.wasb.impl</name>
+ <value>org.apache.hadoop.fs.azure.Wasb</value>
+ <description>AbstractFileSystem implementation class of wasb://</description>
+</property>
+
+<property>
+ <name>fs.AbstractFileSystem.wasbs.impl</name>
+ <value>org.apache.hadoop.fs.azure.Wasbs</value>
+ <description>AbstractFileSystem implementation class of wasbs://</description>
+</property>
+
+<property>
<name>fs.wasb.impl</name>
<value>org.apache.hadoop.fs.azure.NativeAzureFileSystem</value>
<description>The implementation class of the Native Azure Filesystem</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d58ea7c9/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
index 069f17a..924bc3e 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
@@ -173,10 +173,6 @@ public class AbfsConfiguration{
}
}
- public boolean isEmulator() {
- return this.getConfiguration().getBoolean(FS_AZURE_EMULATOR_ENABLED, false);
- }
-
public boolean isSecureMode() {
return isSecure;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d58ea7c9/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
index fc60127..6542a64 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
@@ -77,6 +77,7 @@ import org.apache.hadoop.fs.azurebfs.services.AbfsRestOperation;
import org.apache.hadoop.fs.azurebfs.services.AuthType;
import org.apache.hadoop.fs.azurebfs.services.ExponentialRetryPolicy;
import org.apache.hadoop.fs.azurebfs.services.SharedKeyCredentials;
+import org.apache.hadoop.fs.azurebfs.utils.UriUtils;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsAction;
@@ -86,6 +87,7 @@ import org.apache.http.client.utils.URIBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_ABFS_ENDPOINT;
import static org.apache.hadoop.util.Time.now;
/**
@@ -146,7 +148,27 @@ public class AzureBlobFileSystemStore {
final URIBuilder uriBuilder = new URIBuilder();
uriBuilder.setScheme(scheme);
- uriBuilder.setHost(hostName);
+
+ // For testing purposes, an IP address and port may be provided to override
+ // the host specified in the FileSystem URI. Also note that the format of
+ // the Azure Storage Service URI changes from
+ // http[s]://[account][domain-suffix]/[filesystem] to
+ // http[s]://[ip]:[port]/[account]/[filesystem].
+ String endPoint = abfsConfiguration.getConfiguration().get(AZURE_ABFS_ENDPOINT);
+ if (endPoint == null || !endPoint.contains(AbfsHttpConstants.COLON)) {
+ uriBuilder.setHost(hostName);
+ return uriBuilder;
+ }
+
+ // Split ip and port
+ String[] data = endPoint.split(AbfsHttpConstants.COLON);
+ if (data.length != 2) {
+ throw new RuntimeException(String.format("ABFS endpoint is not set correctly : %s, "
+ + "Do not specify scheme when using {IP}:{PORT}", endPoint));
+ }
+ uriBuilder.setHost(data[0].trim());
+ uriBuilder.setPort(Integer.parseInt(data[1].trim()));
+ uriBuilder.setPath("/" + UriUtils.extractAccountNameFromHostName(hostName));
return uriBuilder;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d58ea7c9/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java
index 8ad0dc6..ca4c9c3 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java
@@ -37,9 +37,6 @@ public final class ConfigurationKeys {
public static final String AZURE_BACKOFF_INTERVAL = "fs.azure.io.retry.backoff.interval";
public static final String AZURE_MAX_IO_RETRIES = "fs.azure.io.retry.max.retries";
- // Remove this and use common azure storage emulator property for public release.
- public static final String FS_AZURE_EMULATOR_ENABLED = "fs.azure.abfs.emulator.enabled";
-
// Read and write buffer sizes defined by the user
public static final String AZURE_WRITE_BUFFER_SIZE = "fs.azure.write.request.size";
public static final String AZURE_READ_BUFFER_SIZE = "fs.azure.read.request.size";
@@ -60,6 +57,8 @@ public final class ConfigurationKeys {
public static final String AZURE_KEY_ACCOUNT_KEYPROVIDER_PREFIX = "fs.azure.account.keyprovider.";
public static final String AZURE_KEY_ACCOUNT_SHELLKEYPROVIDER_SCRIPT = "fs.azure.shellkeyprovider.script";
+ /** End point of ABFS account: {@value}. */
+ public static final String AZURE_ABFS_ENDPOINT = "fs.azure.abfs.endpoint";
/** Prefix for auth type properties: {@value}. */
public static final String FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME = "fs.azure.account.auth.type.";
/** Prefix for oauth token provider type: {@value}. */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d58ea7c9/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java
index 7652adf..1bbc1b3 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java
@@ -41,20 +41,21 @@ public final class UriUtils {
}
/**
- * Extracts the raw account name from account name.
- * @param accountName to extract the raw account name.
- * @return extracted raw account name.
+ * Extracts the account name from the host name.
+ * @param hostName the fully-qualified domain name of the storage service
+ * endpoint (e.g. {account}.dfs.core.windows.net.
+ * @return the storage service account name.
*/
- public static String extractRawAccountFromAccountName(final String accountName) {
- if (accountName == null || accountName.isEmpty()) {
+ public static String extractAccountNameFromHostName(final String hostName) {
+ if (hostName == null || hostName.isEmpty()) {
return null;
}
- if (!containsAbfsUrl(accountName)) {
+ if (!containsAbfsUrl(hostName)) {
return null;
}
- String[] splitByDot = accountName.split("\\.");
+ String[] splitByDot = hostName.split("\\.");
if (splitByDot.length == 0) {
return null;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d58ea7c9/hadoop-tools/hadoop-azure/src/site/markdown/testing_azure.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/testing_azure.md b/hadoop-tools/hadoop-azure/src/site/markdown/testing_azure.md
index c148807..a4d920a 100644
--- a/hadoop-tools/hadoop-azure/src/site/markdown/testing_azure.md
+++ b/hadoop-tools/hadoop-azure/src/site/markdown/testing_azure.md
@@ -90,7 +90,7 @@ For example:
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
<property>
- <name>fs.azure.test.account.name</name>
+ <name>fs.azure.wasb.account.name</name>
<value>{ACCOUNTNAME}.blob.core.windows.net</value>
</property>
<property>
@@ -126,7 +126,7 @@ Overall, to run all the tests using `mvn test`, a sample `azure-auth-keys.xml`
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
<property>
- <name>fs.azure.test.account.name</name>
+ <name>fs.azure.wasb.account.name</name>
<value>{ACCOUNTNAME}.blob.core.windows.net</value>
</property>
<property>
@@ -576,77 +576,172 @@ This will delete the containers; the output log of the test run will
provide the details and summary of the operation.
-## Testing ABFS
+## Testing the Azure ABFS Client
-The ABFS Connector tests share the same account as the wasb tests; this is
-needed for cross-connector compatibility tests.
-
-This makes for a somewhat complex set of configuration options.
-
-Here are the settings for an account `ACCOUNTNAME`
+Azure Data Lake Storage Gen 2 (ADLS Gen 2) is a set of capabilities dedicated to
+big data analytics, built on top of Azure Blob Storage. The ABFS and ABFSS
+schemes target the ADLS Gen 2 REST API, and the WASB and WASBS schemes target
+the Azure Blob Storage REST API. ADLS Gen 2 offers better performance and
+scalability. ADLS Gen 2 also offers authentication and authorization compatible
+with the Hadoop Distributed File System permissions model when hierarchical
+namespace is enabled for the storage account. Furthermore, the metadata and data
+produced by ADLS Gen 2 REST API can be consumed by Blob REST API, and vice versa.
+
+In order to test ABFS, please add the following configuration to your
+`src/test/resources/azure-auth-keys.xml` file. Note that the ABFS tests include
+compatibility tests which require WASB credentials, in addition to the ABFS
+credentials.
```xml
-<property>
- <name>abfs.account.name</name>
- <value>ACCOUNTNAME</value>
-</property>
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<configuration xmlns:xi="http://www.w3.org/2001/XInclude">
+ <property>
+ <name>fs.azure.abfs.account.name</name>
+ <value>{ACCOUNT_NAME}.dfs.core.windows.net</value>
+ </property>
-<property>
- <name>abfs.account.full.name</name>
- <value>${abfs.account.name}.dfs.core.windows.net</value>
-</property>
+ <property>
+ <name>fs.azure.account.key.{ACCOUNT_NAME}.dfs.core.windows.net</name>
+ <value>{ACCOUNT_ACCESS_KEY}</value>
+ </property>
-<property>
- <name>abfs.account.key</name>
- <value>SECRETKEY==</value>
-</property>
+ <property>
+ <name>fs.azure.wasb.account.name</name>
+ <value>{ACCOUNT_NAME}.blob.core.windows.net</value>
+ </property>
+
+ <property>
+ <name>fs.azure.account.key.{ACCOUNT_NAME}.blob.core.windows.net</name>
+ <value>{ACCOUNT_ACCESS_KEY}</value>
+ </property>
-<property>
- <name>fs.azure.account.key.ACCOUNTNAME.dfs.core.windows.net</name>
- <value>${abfs.account.key}</value>
-</property>
+ <property>
+ <name>fs.contract.test.fs.abfs</name>
+ <value>abfs://{CONTAINER_NAME}@{ACCOUNT_NAME}.dfs.core.windows.net</value>
+ <description>A file system URI to be used by the contract tests.</description>
+ </property>
-<property>
- <name>fs.azure.account.key.ACCOUNTNAME.blob.core.windows.net</name>
- <value>${abfs.account.key}</value>
-</property>
+ <property>
+ <name>fs.contract.test.fs.wasb</name>
+ <value>wasb://{CONTAINER_NAME}@{ACCOUNT_NAME}.blob.core.windows.net</value>
+ <description>A file system URI to be used by the contract tests.</description>
+ </property>
+</configuration>
+```
-<property>
- <name>fs.azure.test.account.key.ACCOUNTNAME.dfs.core.windows.net</name>
- <value>${abfs.account.key}</value>
-</property>
+To run OAuth and ACL test cases you must use a storage account with the
+hierarchical namespace enabled, and set the following configuration settings:
+```xml
+<!--=========================== AUTHENTICATION OPTIONS ===================-->
+<!--ATTENTION:
+ TO RUN ABFS & WASB COMPATIBILITY TESTS, YOU MUST SET AUTH TYPE AS SharedKey.
+ OAUTH IS INTRODUCED TO ABFS ONLY.-->
<property>
- <name>fs.azure.test.account.key.ACCOUNTNAME.blob.core.windows.net</name>
- <value>${abfs.account.key}</value>
+ <name>fs.azure.account.auth.type.{YOUR_ABFS_ACCOUNT_NAME}</name>
+ <value>{AUTH TYPE}</value>
+ <description>The authorization type can be SharedKey, OAuth, or Custom. The
+ default is SharedKey.</description>
</property>
-<property>
- <name>fs.azure.account.key.ACCOUNTNAME</name>
- <value>${abfs.account.key}</value>
-</property>
+<!--============================= FOR OAUTH ===========================-->
+<!--IF AUTH TYPE IS SET AS OAUTH, FOLLOW THE STEPS BELOW-->
+<!--NOTICE: AAD client and tenant related properties can be obtained through Azure Portal-->
-<property>
- <name>fs.azure.test.account.key.ACCOUNTNAME</name>
- <value>${abfs.account.key}</value>
-</property>
+ <!--1. UNCOMMENT BELOW AND CHOOSE YOUR OAUTH PROVIDER TYPE -->
-<property>
- <name>fs.azure.test.account.name</name>
- <value>${abfs.account.full.name}</value>
-</property>
+ <!--
+ <property>
+ <name>fs.azure.account.oauth.provider.type.{ABFS_ACCOUNT_NAME}</name>
+ <value>org.apache.hadoop.fs.azurebfs.oauth2.{Token Provider Class name}</value>
+ <description>The full name of token provider class name.</description>
+ </property>
+ -->
-<property>
- <name>fs.contract.test.fs.abfs</name>
- <value>abfs://TESTCONTAINER@ACCOUNTNAME.dfs.core.windows.net</value>
- <description>Container for contract tests</description>
-</property>
+ <!--2. UNCOMMENT BELOW AND SET CREDENTIALS ACCORDING TO THE PROVIDER TYPE-->
-<property>
- <name>fs.contract.test.fs.abfss</name>
- <value>abfss://TESTCONTAINER@ACCOUNTNAME.dfs.core.windows.net</value>
- <description>Container for contract tests</description>
-</property>
+ <!--2.1. If "ClientCredsTokenProvider" is set as key provider, uncomment below and
+ set auth endpoint, client id and secret below-->
+ <!--
+ <property>
+ <name>fs.azure.account.oauth2.client.endpoint.{ABFS_ACCOUNT_NAME}</name>
+ <value>https://login.microsoftonline.com/{TENANTID}/oauth2/token</value>
+ <description>Token end point, this can be found through Azure portal</description>
+ </property>
+ <property>
+ <name>fs.azure.account.oauth2.client.id.{ABFS_ACCOUNT_NAME}</name>
+ <value>{client id}</value>
+ <description>AAD client id.</description>
+ </property>
+
+ <property>
+ <name>fs.azure.account.oauth2.client.secret.{ABFS_ACCOUNT_NAME}</name>
+ <value>{client secret}</value>
+ </property>
+ -->
+
+ <!--2.2. If "UserPasswordTokenProvider" is set as key provider, uncomment below and
+ set auth endpoint, use name and password-->
+ <!--
+ <property>
+ <name>fs.azure.account.oauth2.client.endpoint.{ABFS_ACCOUNT_NAME}</name>
+ <value>https://login.microsoftonline.com/{TENANTID}/oauth2/token</value>
+ <description>Token end point, this can be found through Azure portal</description>
+ </property>
+ <property>
+ <name>fs.azure.account.oauth2.user.name.{ABFS_ACCOUNT_NAME}</name>
+ <value>{user name}</value>
+ </property>
+
+ <property>
+ <name>fs.azure.account.oauth2.user.password.{ABFS_ACCOUNT_NAME}</name>
+ <value>{user password}</value>
+ </property>
+ -->
+
+ <!--2.3. If "MsiTokenProvider" is set as key provider, uncomment below and
+ set tenantGuid and client id.-->
+ <!--
+ <property>
+ <name>fs.azure.account.oauth2.msi.tenant.{ABFS_ACCOUNT_NAME}</name>
+ <value>{tenantGuid}</value>
+ <description>msi tenantGuid.</description>
+ </property>
+
+ <property>
+ <name>fs.azure.account.oauth2.client.id.{ABFS_ACCOUNT_NAME}</name>
+ <value>{client id}</value>
+ <description>AAD client id.</description>
+ </property>
+ -->
+
+ <!--2.4. If "RefreshTokenBasedTokenProvider" is set as key provider, uncomment below and
+ set refresh token and client id.-->
+ <!--
+ <property>
+ <name>fs.azure.account.oauth2.refresh.token.{ABFS_ACCOUNT_NAME}</name>
+ <value>{refresh token}</value>
+ <description>refresh token.</description>
+ </property>
+
+ <property>
+ <name>fs.azure.account.oauth2.client.id.{ABFS_ACCOUNT_NAME}</name>
+ <value>{client id}</value>
+ <description>AAD client id.</description>
+ </property>
+ -->
```
+
+If running tests against an endpoint that uses the URL format
+http[s]://[ip]:[port]/[account]/[filesystem] instead of
+http[s]://[account][domain-suffix]/[filesystem], please use the following:
+
+```xml
+<property>
+ <name>fs.azure.abfs.endpoint</name>
+ <value>{IP}:{PORT}</value>
+</property>
+```
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d58ea7c9/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java
index 5b36c87..b65ce78 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java
@@ -48,6 +48,7 @@ import java.util.concurrent.ConcurrentLinkedQueue;
import static org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.DEFAULT_STORAGE_EMULATOR_ACCOUNT_NAME;
import static org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.KEY_USE_LOCAL_SAS_KEY_MODE;
import static org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.KEY_USE_SECURE_MODE;
+import static org.apache.hadoop.fs.azure.integration.AzureTestUtils.verifyWasbAccountNameInConfig;
/**
* Helper class to create WASB file systems backed by either a mock in-memory
@@ -58,11 +59,14 @@ public final class AzureBlobStorageTestAccount implements AutoCloseable,
private static final Logger LOG = LoggerFactory.getLogger(
AzureBlobStorageTestAccount.class);
- private static final String ACCOUNT_KEY_PROPERTY_NAME = "fs.azure.account.key.";
private static final String SAS_PROPERTY_NAME = "fs.azure.sas.";
private static final String TEST_CONFIGURATION_FILE_NAME = "azure-test.xml";
- private static final String TEST_ACCOUNT_NAME_PROPERTY_NAME = "fs.azure.test.account.name";
+ public static final String ACCOUNT_KEY_PROPERTY_NAME = "fs.azure.account.key.";
+ public static final String TEST_ACCOUNT_NAME_PROPERTY_NAME = "fs.azure.account.name";
+ public static final String WASB_TEST_ACCOUNT_NAME_WITH_DOMAIN = "fs.azure.wasb.account.name";
public static final String MOCK_ACCOUNT_NAME = "mockAccount.blob.core.windows.net";
+ public static final String WASB_ACCOUNT_NAME_DOMAIN_SUFFIX = ".blob.core.windows.net";
+ public static final String WASB_ACCOUNT_NAME_DOMAIN_SUFFIX_REGEX = "\\.blob(\\.preprod)?\\.core\\.windows\\.net";
public static final String MOCK_CONTAINER_NAME = "mockContainer";
public static final String WASB_AUTHORITY_DELIMITER = "@";
public static final String WASB_SCHEME = "wasb";
@@ -379,7 +383,7 @@ public final class AzureBlobStorageTestAccount implements AutoCloseable,
containerName);
container.create();
- String accountName = conf.get(TEST_ACCOUNT_NAME_PROPERTY_NAME);
+ String accountName = verifyWasbAccountNameInConfig(conf);
// Ensure that custom throttling is disabled and tolerate concurrent
// out-of-band appends.
@@ -525,7 +529,7 @@ public final class AzureBlobStorageTestAccount implements AutoCloseable,
static CloudStorageAccount createTestAccount(Configuration conf)
throws URISyntaxException, KeyProviderException {
- String testAccountName = conf.get(TEST_ACCOUNT_NAME_PROPERTY_NAME);
+ String testAccountName = verifyWasbAccountNameInConfig(conf);
if (testAccountName == null) {
LOG.warn("Skipping live Azure test because of missing test account");
return null;
@@ -570,16 +574,16 @@ public final class AzureBlobStorageTestAccount implements AutoCloseable,
String containerName = useContainerSuffixAsContainerName
? containerNameSuffix
: String.format(
- "wasbtests-%s-%tQ%s",
+ "wasbtests-%s-%s%s",
System.getProperty("user.name"),
- new Date(),
+ UUID.randomUUID().toString(),
containerNameSuffix);
container = account.createCloudBlobClient().getContainerReference(
containerName);
if (createOptions.contains(CreateOptions.CreateContainer)) {
container.createIfNotExists();
}
- String accountName = conf.get(TEST_ACCOUNT_NAME_PROPERTY_NAME);
+ String accountName = verifyWasbAccountNameInConfig(conf);
if (createOptions.contains(CreateOptions.UseSas)) {
String sas = generateSAS(container,
createOptions.contains(CreateOptions.Readonly));
@@ -741,7 +745,7 @@ public final class AzureBlobStorageTestAccount implements AutoCloseable,
CloudBlobClient blobClient = account.createCloudBlobClient();
// Capture the account URL and the account name.
- String accountName = conf.get(TEST_ACCOUNT_NAME_PROPERTY_NAME);
+ String accountName = verifyWasbAccountNameInConfig(conf);
configureSecureModeTestSettings(conf);
@@ -814,7 +818,7 @@ public final class AzureBlobStorageTestAccount implements AutoCloseable,
CloudBlobClient blobClient = account.createCloudBlobClient();
// Capture the account URL and the account name.
- String accountName = conf.get(TEST_ACCOUNT_NAME_PROPERTY_NAME);
+ String accountName = verifyWasbAccountNameInConfig(conf);
configureSecureModeTestSettings(conf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d58ea7c9/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestFileSystemOperationExceptionMessage.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestFileSystemOperationExceptionMessage.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestFileSystemOperationExceptionMessage.java
index 6d5e72e..af570bd 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestFileSystemOperationExceptionMessage.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestFileSystemOperationExceptionMessage.java
@@ -29,6 +29,7 @@ import com.microsoft.azure.storage.CloudStorageAccount;
import org.junit.Test;
import static org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.NO_ACCESS_TO_CONTAINER_MSG;
+import static org.apache.hadoop.fs.azure.integration.AzureTestUtils.verifyWasbAccountNameInConfig;
/**
* Test for error messages coming from SDK.
@@ -46,7 +47,7 @@ public class ITestFileSystemOperationExceptionMessage
AzureBlobStorageTestAccount.createTestAccount(conf);
AzureTestUtils.assume("No test account", account != null);
- String testStorageAccount = conf.get("fs.azure.test.account.name");
+ String testStorageAccount = verifyWasbAccountNameInConfig(conf);
conf = new Configuration();
conf.set("fs.AbstractFileSystem.wasb.impl",
"org.apache.hadoop.fs.azure.Wasb");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d58ea7c9/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestWasbUriAndConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestWasbUriAndConfiguration.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestWasbUriAndConfiguration.java
index bee0220..7783684 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestWasbUriAndConfiguration.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestWasbUriAndConfiguration.java
@@ -500,32 +500,6 @@ public class ITestWasbUriAndConfiguration extends AbstractWasbTestWithTimeout {
}
@Test
- public void testNoAbstractFileSystemImplementationSpecifiedForWasbsScheme() throws Exception {
- try {
- testAccount = AzureBlobStorageTestAccount.createMock();
- Configuration conf = testAccount.getFileSystem().getConf();
- String authority = testAccount.getFileSystem().getUri().getAuthority();
- URI defaultUri = new URI("wasbs", authority, null, null, null);
- conf.set(FS_DEFAULT_NAME_KEY, defaultUri.toString());
-
- FileSystem fs = FileSystem.get(conf);
- assertTrue(fs instanceof NativeAzureFileSystem);
- assertEquals("wasbs", fs.getScheme());
-
- // should throw if 'fs.AbstractFileSystem.wasbs.impl'' is not specified
- try{
- FileContext.getFileContext(conf).getDefaultFileSystem();
- fail("Should've thrown.");
- }catch(UnsupportedFileSystemException e){
- }
-
- } finally {
- testAccount.cleanup();
- FileSystem.closeAll();
- }
- }
-
- @Test
public void testCredentialProviderPathExclusions() throws Exception {
String providerPath =
"user:///,jceks://wasb/user/hrt_qa/sqoopdbpasswd.jceks," +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d58ea7c9/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestConstants.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestConstants.java
index 0b72f06..82907c5 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestConstants.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestConstants.java
@@ -143,12 +143,10 @@ public interface AzureTestConstants {
- String ACCOUNT_KEY_PROPERTY_NAME
- = "fs.azure.account.key.";
+ String ACCOUNT_KEY_PROPERTY_NAME = "fs.azure.account.key.";
+ String ACCOUNT_NAME_PROPERTY_NAME = "fs.azure.account.name";
String SAS_PROPERTY_NAME = "fs.azure.sas.";
String TEST_CONFIGURATION_FILE_NAME = "azure-test.xml";
- String TEST_ACCOUNT_NAME_PROPERTY_NAME
- = "fs.azure.test.account.name";
String MOCK_ACCOUNT_NAME
= "mockAccount.blob.core.windows.net";
String MOCK_CONTAINER_NAME = "mockContainer";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d58ea7c9/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestUtils.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestUtils.java
index 8d2a104..b438c8e 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestUtils.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestUtils.java
@@ -36,10 +36,14 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount;
import org.apache.hadoop.fs.azure.NativeAzureFileSystem;
+
+import static org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount.WASB_ACCOUNT_NAME_DOMAIN_SUFFIX_REGEX;
+import static org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount.WASB_TEST_ACCOUNT_NAME_WITH_DOMAIN;
import static org.apache.hadoop.fs.azure.integration.AzureTestConstants.*;
import static org.apache.hadoop.test.MetricsAsserts.getLongCounter;
import static org.apache.hadoop.test.MetricsAsserts.getLongGauge;
import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
+import static org.junit.Assume.assumeTrue;
/**
* Utilities for the Azure tests. Based on {@code S3ATestUtils}, so
@@ -476,4 +480,18 @@ public final class AzureTestUtils extends Assert {
+ KEY_SCALE_TESTS_ENABLED,
enabled);
}
+
+ /**
+ * Check the account name for WASB tests is set correctly and return.
+ */
+ public static String verifyWasbAccountNameInConfig(Configuration conf) {
+ String accountName = conf.get(ACCOUNT_NAME_PROPERTY_NAME);
+ if (accountName == null) {
+ accountName = conf.get(WASB_TEST_ACCOUNT_NAME_WITH_DOMAIN);
+ }
+ assumeTrue("Account for WASB is missing or it is not in correct format",
+ accountName != null && !accountName.endsWith(WASB_ACCOUNT_NAME_DOMAIN_SUFFIX_REGEX));
+ return accountName;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d58ea7c9/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestRollingWindowAverage.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestRollingWindowAverage.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestRollingWindowAverage.java
index 9b1fb8e..cd8b692 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestRollingWindowAverage.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestRollingWindowAverage.java
@@ -18,8 +18,8 @@
package org.apache.hadoop.fs.azure.metrics;
-import static org.junit.Assert.*;
-import org.junit.*;
+import static org.junit.Assert.assertEquals;
+import org.junit.Test;
public class TestRollingWindowAverage {
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d58ea7c9/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java
index e0afeb4..f61954f 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java
@@ -35,7 +35,6 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.azurebfs.services.AuthType;
-import org.apache.hadoop.fs.azure.AbstractWasbTestWithTimeout;
import org.apache.hadoop.fs.azure.AzureNativeFileSystemStore;
import org.apache.hadoop.fs.azure.NativeAzureFileSystem;
import org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation;
@@ -45,12 +44,12 @@ import org.apache.hadoop.fs.azurebfs.utils.UriUtils;
import org.apache.hadoop.fs.contract.ContractTestUtils;
import org.apache.hadoop.io.IOUtils;
+import static org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount.WASB_ACCOUNT_NAME_DOMAIN_SUFFIX;
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.*;
import static org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode.FILE_SYSTEM_NOT_FOUND;
import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.*;
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
import static org.junit.Assume.assumeTrue;
-import static org.hamcrest.CoreMatchers.*;
/**
* Base for AzureBlobFileSystem Integration tests.
@@ -58,12 +57,12 @@ import static org.hamcrest.CoreMatchers.*;
* <I>Important: This is for integration tests only.</I>
*/
public abstract class AbstractAbfsIntegrationTest extends
- AbstractWasbTestWithTimeout {
+ AbstractAbfsTestWithTimeout {
private static final Logger LOG =
LoggerFactory.getLogger(AbstractAbfsIntegrationTest.class);
- private boolean isEmulator;
+ private boolean isIPAddress;
private NativeAzureFileSystem wasb;
private AzureBlobFileSystem abfs;
private String abfsScheme;
@@ -75,27 +74,28 @@ public abstract class AbstractAbfsIntegrationTest extends
private AuthType authType;
protected AbstractAbfsIntegrationTest() {
- fileSystemName = ABFS_TEST_CONTAINER_PREFIX + UUID.randomUUID().toString();
+ fileSystemName = TEST_CONTAINER_PREFIX + UUID.randomUUID().toString();
configuration = new Configuration();
- configuration.addResource(ABFS_TEST_RESOURCE_XML);
- this.accountName = this.configuration.get(FS_AZURE_TEST_ACCOUNT_NAME);
+ configuration.addResource(TEST_CONFIGURATION_FILE_NAME);
+
+ this.accountName = this.configuration.get(FS_AZURE_ACCOUNT_NAME);
+ if (accountName == null) {
+ // check if accountName is set using different config key
+ accountName = configuration.get(FS_AZURE_ABFS_ACCOUNT_NAME);
+ }
+ assumeTrue("Not set: " + FS_AZURE_ABFS_ACCOUNT_NAME,
+ accountName != null && !accountName.isEmpty());
+
authType = configuration.getEnum(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME
+ accountName, AuthType.SharedKey);
abfsScheme = authType == AuthType.SharedKey ? FileSystemUriSchemes.ABFS_SCHEME
: FileSystemUriSchemes.ABFS_SECURE_SCHEME;
- String accountName = configuration.get(FS_AZURE_TEST_ACCOUNT_NAME, "");
- assumeTrue("Not set: " + FS_AZURE_TEST_ACCOUNT_NAME,
- !accountName.isEmpty());
- assertThat("The key in " + FS_AZURE_TEST_ACCOUNT_KEY_PREFIX
- + " is not bound to an ABFS account",
- accountName, containsString("dfs.core.windows.net"));
- String fullKey = FS_AZURE_TEST_ACCOUNT_KEY_PREFIX
- + accountName;
-
if (authType == AuthType.SharedKey) {
- assumeTrue("Not set: " + fullKey, configuration.get(fullKey) != null);
+ String keyProperty = FS_AZURE_ACCOUNT_KEY_PREFIX + accountName;
+ assumeTrue("Not set: " + keyProperty, configuration.get(keyProperty) != null);
+ // Update credentials
} else {
String accessTokenProviderKey = FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME + accountName;
assumeTrue("Not set: " + accessTokenProviderKey, configuration.get(accessTokenProviderKey) != null);
@@ -113,7 +113,17 @@ public abstract class AbstractAbfsIntegrationTest extends
this.testUrl = defaultUri.toString();
configuration.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri.toString());
configuration.setBoolean(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, true);
- this.isEmulator = this.configuration.getBoolean(FS_AZURE_EMULATOR_ENABLED, false);
+ // For testing purposes, an IP address and port may be provided to override
+ // the host specified in the FileSystem URI. Also note that the format of
+ // the Azure Storage Service URI changes from
+ // http[s]://[account][domain-suffix]/[filesystem] to
+ // http[s]://[ip]:[port]/[account]/[filesystem].
+ String endPoint = configuration.get(AZURE_ABFS_ENDPOINT);
+ if (endPoint != null && endPoint.contains(":") && endPoint.split(":").length == 2) {
+ this.isIPAddress = true;
+ } else {
+ this.isIPAddress = false;
+ }
}
@@ -122,13 +132,22 @@ public abstract class AbstractAbfsIntegrationTest extends
//Create filesystem first to make sure getWasbFileSystem() can return an existing filesystem.
createFileSystem();
- if (!isEmulator && authType == AuthType.SharedKey) {
+ if (!isIPAddress && authType == AuthType.SharedKey) {
final URI wasbUri = new URI(abfsUrlToWasbUrl(getTestUrl()));
final AzureNativeFileSystemStore azureNativeFileSystemStore =
new AzureNativeFileSystemStore();
+
+ // update configuration with wasb credentials
+ String accountNameWithoutDomain = accountName.split("\\.")[0];
+ String wasbAccountName = accountNameWithoutDomain + WASB_ACCOUNT_NAME_DOMAIN_SUFFIX;
+ String keyProperty = FS_AZURE_ACCOUNT_KEY_PREFIX + wasbAccountName;
+ if (configuration.get(keyProperty) == null) {
+ configuration.set(keyProperty, getAccountKey());
+ }
+
azureNativeFileSystemStore.initialize(
wasbUri,
- getConfiguration(),
+ configuration,
new AzureFileSystemInstrumentation(getConfiguration()));
wasb = new NativeAzureFileSystem(azureNativeFileSystemStore);
@@ -201,7 +220,9 @@ public abstract class AbstractAbfsIntegrationTest extends
}
protected String getHostName() {
- return configuration.get(FS_AZURE_TEST_HOST_NAME);
+ // READ FROM ENDPOINT, THIS IS CALLED ONLY WHEN TESTING AGAINST DEV-FABRIC
+ String endPoint = configuration.get(AZURE_ABFS_ENDPOINT);
+ return endPoint.split(":")[0];
}
protected void setTestUrl(String testUrl) {
@@ -220,21 +241,21 @@ public abstract class AbstractAbfsIntegrationTest extends
}
protected String getAccountName() {
- return configuration.get(FS_AZURE_TEST_ACCOUNT_NAME);
+ return this.accountName;
}
protected String getAccountKey() {
return configuration.get(
- FS_AZURE_TEST_ACCOUNT_KEY_PREFIX
- + getAccountName());
+ FS_AZURE_ACCOUNT_KEY_PREFIX
+ + accountName);
}
protected Configuration getConfiguration() {
return configuration;
}
- protected boolean isEmulator() {
- return isEmulator;
+ protected boolean isIPAddress() {
+ return isIPAddress;
}
protected AuthType getAuthType() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d58ea7c9/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsTestWithTimeout.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsTestWithTimeout.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsTestWithTimeout.java
new file mode 100644
index 0000000..fee90ab
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsTestWithTimeout.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.azurebfs;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.rules.TestName;
+import org.junit.rules.Timeout;
+
+import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.TEST_TIMEOUT;
+
+/**
+ * Base class for any ABFS test with timeouts & named threads.
+ * This class does not attempt to bind to Azure.
+ */
+public class AbstractAbfsTestWithTimeout extends Assert {
+ /**
+ * The name of the current method.
+ */
+ @Rule
+ public TestName methodName = new TestName();
+ /**
+ * Set the timeout for every test.
+ * This is driven by the value returned by {@link #getTestTimeoutMillis()}.
+ */
+ @Rule
+ public Timeout testTimeout = new Timeout(getTestTimeoutMillis());
+
+ /**
+ * Name the junit thread for the class. This will overridden
+ * before the individual test methods are run.
+ */
+ @BeforeClass
+ public static void nameTestThread() {
+ Thread.currentThread().setName("JUnit");
+ }
+
+ /**
+ * Name the thread to the current test method.
+ */
+ @Before
+ public void nameThread() {
+ Thread.currentThread().setName("JUnit-" + methodName.getMethodName());
+ }
+
+ /**
+ * Override point: the test timeout in milliseconds.
+ * @return a timeout in milliseconds
+ */
+ protected int getTestTimeoutMillis() {
+ return TEST_TIMEOUT;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d58ea7c9/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java
index 800b95a..46ac866 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java
@@ -69,7 +69,7 @@ public class ITestAzureBlobFileSystemBackCompat extends
private String getBlobConnectionString() {
String connectionString;
- if (isEmulator()) {
+ if (isIPAddress()) {
connectionString = "DefaultEndpointsProtocol=http;BlobEndpoint=http://"
+ this.getHostName() + ":8880/" + this.getAccountName().split("\\.") [0]
+ ";AccountName=" + this.getAccountName().split("\\.")[0]
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d58ea7c9/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java
index e4f3ea3..c4bfee2 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java
@@ -50,7 +50,7 @@ public class ITestWasbAbfsCompatibility extends AbstractAbfsIntegrationTest {
LoggerFactory.getLogger(ITestWasbAbfsCompatibility.class);
public ITestWasbAbfsCompatibility() throws Exception {
- Assume.assumeFalse("Emulator is not supported", isEmulator());
+ Assume.assumeFalse("Emulator is not supported", isIPAddress());
Assume.assumeTrue(this.getAuthType() == AuthType.SharedKey);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d58ea7c9/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsConfigurationFieldsValidation.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsConfigurationFieldsValidation.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsConfigurationFieldsValidation.java
index 556dcdb..bedd4f3 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsConfigurationFieldsValidation.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsConfigurationFieldsValidation.java
@@ -23,6 +23,7 @@ import java.lang.reflect.Field;
import org.apache.commons.codec.Charsets;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys;
+import org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys;
import org.apache.hadoop.fs.azurebfs.contracts.annotations.ConfigurationValidationAnnotations.IntegerConfigurationValidatorAnnotation;
import org.apache.hadoop.fs.azurebfs.contracts.annotations.ConfigurationValidationAnnotations.BooleanConfigurationValidatorAnnotation;
import org.apache.hadoop.fs.azurebfs.contracts.annotations.ConfigurationValidationAnnotations.StringConfigurationValidatorAnnotation;
@@ -98,7 +99,7 @@ public class TestAbfsConfigurationFieldsValidation {
this.encodedString = new String(base64.encode("base64Value".getBytes(Charsets.UTF_8)), Charsets.UTF_8);
this.encodedAccountKey = new String(base64.encode("someAccountKey".getBytes(Charsets.UTF_8)), Charsets.UTF_8);
Configuration configuration = new Configuration();
- configuration.addResource("azure-bfs-test.xml");
+ configuration.addResource(TestConfigurationKeys.TEST_CONFIGURATION_FILE_NAME);
configuration.set(INT_KEY, "1234565");
configuration.set(LONG_KEY, "4194304");
configuration.set(STRING_KEY, "stringValue");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d58ea7c9/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java
index 67301c7..7852f25 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java
@@ -22,10 +22,9 @@ package org.apache.hadoop.fs.azurebfs.constants;
* Responsible to keep all the Azure Blob File System configurations keys in Hadoop configuration file.
*/
public final class TestConfigurationKeys {
- public static final String FS_AZURE_TEST_ACCOUNT_NAME = "fs.azure.test.account.name";
- public static final String FS_AZURE_TEST_ACCOUNT_KEY_PREFIX = "fs.azure.test.account.key.";
- public static final String FS_AZURE_TEST_HOST_NAME = "fs.azure.test.host.name";
- public static final String FS_AZURE_TEST_HOST_PORT = "fs.azure.test.host.port";
+ public static final String FS_AZURE_ACCOUNT_NAME = "fs.azure.account.name";
+ public static final String FS_AZURE_ABFS_ACCOUNT_NAME = "fs.azure.abfs.account.name";
+ public static final String FS_AZURE_ACCOUNT_KEY_PREFIX = "fs.azure.account.key.";
public static final String FS_AZURE_CONTRACT_TEST_URI = "fs.contract.test.fs.abfs";
public static final String FS_AZURE_BLOB_DATA_CONTRIBUTOR_CLIENT_ID = "fs.azure.account.oauth2.contributor.client.id";
@@ -34,9 +33,9 @@ public final class TestConfigurationKeys {
public static final String FS_AZURE_BLOB_DATA_READER_CLIENT_ID = "fs.azure.account.oauth2.reader.client.id";
public static final String FS_AZURE_BLOB_DATA_READER_CLIENT_SECRET = "fs.azure.account.oauth2.reader.client.secret";
- public static final String ABFS_TEST_RESOURCE_XML = "azure-bfs-test.xml";
-
- public static final String ABFS_TEST_CONTAINER_PREFIX = "abfs-testcontainer-";
+ public static final String TEST_CONFIGURATION_FILE_NAME = "azure-test.xml";
+ public static final String TEST_CONTAINER_PREFIX = "abfs-testcontainer-";
+ public static final int TEST_TIMEOUT = 10 * 60 * 1000;
private TestConfigurationKeys() {}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d58ea7c9/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ABFSContractTestBinding.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ABFSContractTestBinding.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ABFSContractTestBinding.java
index 5505e6a..6d089d1 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ABFSContractTestBinding.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ABFSContractTestBinding.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest;
import org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes;
import org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys;
import org.apache.hadoop.fs.azurebfs.services.AuthType;
+import org.junit.Assume;
/**
* Bind ABFS contract tests to the Azure test setup/teardown.
@@ -42,6 +43,8 @@ public class ABFSContractTestBinding extends AbstractAbfsIntegrationTest {
if (useExistingFileSystem) {
Configuration configuration = getConfiguration();
String testUrl = configuration.get(TestConfigurationKeys.FS_AZURE_CONTRACT_TEST_URI);
+ Assume.assumeTrue("Contract tests are skipped because of missing config property :"
+ + TestConfigurationKeys.FS_AZURE_CONTRACT_TEST_URI, testUrl != null);
if (getAuthType() != AuthType.SharedKey) {
testUrl = testUrl.replaceFirst(FileSystemUriSchemes.ABFS_SCHEME, FileSystemUriSchemes.ABFS_SECURE_SCHEME);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d58ea7c9/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/diagnostics/TestConfigurationValidators.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/diagnostics/TestConfigurationValidators.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/diagnostics/TestConfigurationValidators.java
index e9e90c9..a9556ab 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/diagnostics/TestConfigurationValidators.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/diagnostics/TestConfigurationValidators.java
@@ -32,7 +32,6 @@ import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.M
import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_READ_BUFFER_SIZE;
import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_WRITE_BUFFER_SIZE;
-import static org.junit.Assert.assertEquals;
/**
* Test configuration validators.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d58ea7c9/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/AbfsTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/AbfsTestUtils.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/AbfsTestUtils.java
new file mode 100644
index 0000000..9c636a2
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/AbfsTestUtils.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.azurebfs.utils;
+
+import com.microsoft.azure.storage.CloudStorageAccount;
+import com.microsoft.azure.storage.blob.CloudBlobClient;
+import com.microsoft.azure.storage.blob.CloudBlobContainer;
+
+import org.junit.Assume;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount;
+import org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest;
+import org.apache.hadoop.fs.azurebfs.services.AuthType;
+
+import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.TEST_CONTAINER_PREFIX;
+
+ /*
+ * Some Utils for ABFS tests.
+ * */
+public final class AbfsTestUtils extends AbstractAbfsIntegrationTest{
+ private static final Logger LOG =
+ LoggerFactory.getLogger(AbfsTestUtils.class);
+ /**
+ * If unit tests were interrupted and crushed accidentally, the test containers won't be deleted.
+ * In that case, dev can use this tool to list and delete all test containers.
+ * By default, all test container used in E2E tests sharing same prefix: "abfs-testcontainer-"
+ */
+
+ public void checkContainers() throws Throwable {
+ Assume.assumeTrue(this.getAuthType() == AuthType.SharedKey);
+ int count = 0;
+ CloudStorageAccount storageAccount = AzureBlobStorageTestAccount.createTestAccount();
+ CloudBlobClient blobClient = storageAccount.createCloudBlobClient();
+ Iterable<CloudBlobContainer> containers
+ = blobClient.listContainers(TEST_CONTAINER_PREFIX);
+ for (CloudBlobContainer container : containers) {
+ count++;
+ LOG.info("Container {}, URI {}",
+ container.getName(),
+ container.getUri());
+ }
+ LOG.info("Found {} test containers", count);
+ }
+
+
+ public void deleteContainers() throws Throwable {
+ Assume.assumeTrue(this.getAuthType() == AuthType.SharedKey);
+ int count = 0;
+ CloudStorageAccount storageAccount = AzureBlobStorageTestAccount.createTestAccount();
+ CloudBlobClient blobClient = storageAccount.createCloudBlobClient();
+ Iterable<CloudBlobContainer> containers
+ = blobClient.listContainers(TEST_CONTAINER_PREFIX);
+ for (CloudBlobContainer container : containers) {
+ LOG.info("Container {} URI {}",
+ container.getName(),
+ container.getUri());
+ if (container.deleteIfExists()) {
+ count++;
+ }
+ }
+ LOG.info("Deleted {} test containers", count);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d58ea7c9/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/CleanUpAbfsTestContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/CleanUpAbfsTestContainer.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/CleanUpAbfsTestContainer.java
deleted file mode 100644
index ef4ddb9..0000000
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/CleanUpAbfsTestContainer.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.azurebfs.utils;
-
-import com.microsoft.azure.storage.CloudStorageAccount;
-import com.microsoft.azure.storage.blob.CloudBlobClient;
-import com.microsoft.azure.storage.blob.CloudBlobContainer;
-
-import org.junit.Assume;
-import org.junit.Test;
-
-import org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount;
-import org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest;
-import org.apache.hadoop.fs.azurebfs.services.AuthType;
-
-import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.ABFS_TEST_CONTAINER_PREFIX;
-
-/**
- * If unit tests were interrupted and crushed accidentally, the test containers won't be deleted.
- * In that case, dev can use this tool to list and delete all test containers.
- * By default, all test container used in E2E tests sharing same prefix: "abfs-testcontainer-"
- */
-public class CleanUpAbfsTestContainer extends AbstractAbfsIntegrationTest{
-
- public CleanUpAbfsTestContainer() {
- Assume.assumeTrue(this.getAuthType() == AuthType.SharedKey);
- }
-
- @Test
- public void testEnumContainers() throws Throwable {
- int count = 0;
- CloudStorageAccount storageAccount = AzureBlobStorageTestAccount.createTestAccount();
- CloudBlobClient blobClient = storageAccount.createCloudBlobClient();
- Iterable<CloudBlobContainer> containers
- = blobClient.listContainers(ABFS_TEST_CONTAINER_PREFIX);
- for (CloudBlobContainer container : containers) {
- count++;
- System.out.println(String.format("Container %s URI %s",
- container.getName(),
- container.getUri()));
- }
- System.out.println(String.format("Found %d test containers", count));
- }
-
- @Test
- public void testDeleteContainers() throws Throwable {
- int count = 0;
- CloudStorageAccount storageAccount = AzureBlobStorageTestAccount.createTestAccount();
- CloudBlobClient blobClient = storageAccount.createCloudBlobClient();
- Iterable<CloudBlobContainer> containers
- = blobClient.listContainers(ABFS_TEST_CONTAINER_PREFIX);
- for (CloudBlobContainer container : containers) {
- System.out.println(String.format("Container %s URI %s",
- container.getName(),
- container.getUri()));
- if (container.deleteIfExists()) {
- count++;
- }
- }
- System.out.println(String.format("Deleted %s test containers", count));
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d58ea7c9/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestUriUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestUriUtils.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestUriUtils.java
index e71e31d..690e56c 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestUriUtils.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestUriUtils.java
@@ -38,11 +38,11 @@ public final class TestUriUtils {
@Test
public void testExtractRawAccountName() throws Exception {
- Assert.assertEquals("abfs", UriUtils.extractRawAccountFromAccountName("abfs.dfs.core.windows.net"));
- Assert.assertEquals("abfs", UriUtils.extractRawAccountFromAccountName("abfs.dfs.preprod.core.windows.net"));
- Assert.assertEquals(null, UriUtils.extractRawAccountFromAccountName("abfs.dfs.cores.windows.net"));
- Assert.assertEquals(null, UriUtils.extractRawAccountFromAccountName(""));
- Assert.assertEquals(null, UriUtils.extractRawAccountFromAccountName(null));
- Assert.assertEquals(null, UriUtils.extractRawAccountFromAccountName("abfs.dfs.cores.windows.net"));
+ Assert.assertEquals("abfs", UriUtils.extractAccountNameFromHostName("abfs.dfs.core.windows.net"));
+ Assert.assertEquals("abfs", UriUtils.extractAccountNameFromHostName("abfs.dfs.preprod.core.windows.net"));
+ Assert.assertEquals(null, UriUtils.extractAccountNameFromHostName("abfs.dfs.cores.windows.net"));
+ Assert.assertEquals(null, UriUtils.extractAccountNameFromHostName(""));
+ Assert.assertEquals(null, UriUtils.extractAccountNameFromHostName(null));
+ Assert.assertEquals(null, UriUtils.extractAccountNameFromHostName("abfs.dfs.cores.windows.net"));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d58ea7c9/hadoop-tools/hadoop-azure/src/test/resources/azure-bfs-test.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/resources/azure-bfs-test.xml b/hadoop-tools/hadoop-azure/src/test/resources/azure-bfs-test.xml
deleted file mode 100644
index 9b908b0..0000000
--- a/hadoop-tools/hadoop-azure/src/test/resources/azure-bfs-test.xml
+++ /dev/null
@@ -1,188 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<configuration xmlns:xi="http://www.w3.org/2001/XInclude">
-
- <!--
- <property>
- <name>fs.azure.test.account.name</name>
- <value>{YOURACCOUNT}</value>
- </property>
-
- <property>
- <name>fs.azure.account.auth.type.{YOURACCOUNT}.dfs.core.windows.net</name>
- <value>{AUTH TYPE}</value>
- <description>The auth type can be : SharedKey, OAuth, Custom. By default "SharedKey" is used.</description>
- </property>
-
- <property>
- <name>fs.contract.test.fs.abfs</name>
- <value>abfs://{CONTAINERNAME}@{ACCOUNTNAME}.dfs.core.windows.net/value>
- <description>The name of the azure file system for testing.</description>
- </property>
-
- <property>
- <name>fs.contract.test.fs.abfss</name>
- <value>abfss://{CONTAINERNAME}@{ACCOUNTNAME}.dfs.core.windows.net/value>
- <description>The name of the azure file system for testing.</description>
- </property>
- -->
-
- <!--If auth type is "SharedKey", provide SharedKey credentials below -->
- <!--
- <property>
- <name>fs.azure.account.key.{YOURACCOUNT}.dfs.core.windows.net</name>
- <value>{ACCOUNTKEY}</value>
- </property>
-
- <property>
- <name>fs.azure.test.account.key.{YOURACCOUNT}.dfs.core.windows.net</name>
- <value>{ACCOUNTKEY}</value>
- </property>
- -->
-
- <!--If auth type is "OAuth", set below properties, AAD client and tenant related properties can be obtained through Azure Portal-->
- <!--
- <property>
- <name>fs.azure.account.oauth.provider.type.{YOURACCOUNT}.dfs.core.windows.net</name>
- <value>org.apache.hadoop.fs.azurebfs.oauth2.{Token Provider Class name}</value>
- <description>The full name of token provider class name.</description>
- </property>
- -->
-
- <!--If "ClientCredsTokenProvider" is set as key provider, set auth endpoint, client id and secret below-->
- <!--
- <property>
- <name>fs.azure.account.oauth2.client.endpoint.{YOURACCOUNT}.dfs.core.windows.net</name>
- <value>https://login.microsoftonline.com/{TENANTID}/oauth2/token</value>
- <description>Token end point, this can be found through Azure portal</description>
- </property>
-
- <property>
- <name>fs.azure.account.oauth2.client.id.{YOURACCOUNT}.dfs.core.windows.net</name>
- <value>{client id}</value>
- <description>AAD client id.</description>
- </property>
-
- <property>
- <name>fs.azure.account.oauth2.client.secret.{YOURACCOUNT}.dfs.core.windows.net</name>
- <value>{client secret}</value>
- </property>
- -->
-
- <!--If "UserPasswordTokenProvider" is set as key provider, set auth endpoint, use name and password below-->
- <!--
- <property>
- <name>fs.azure.account.oauth2.client.endpoint.{YOURACCOUNT}.dfs.core.windows.net</name>
- <value>https://login.microsoftonline.com/{TENANTID}/oauth2/token</value>
- <description>Token end point, this can be found through Azure portal</description>
- </property>
-
- <property>
- <name>fs.azure.account.oauth2.user.name.{YOURACCOUNT}.dfs.core.windows.net</name>
- <value>{user name}</value>
- </property>
-
- <property>
- <name>fs.azure.account.oauth2.user.password.{YOURACCOUNT}.dfs.core.windows.net</name>
- <value>{user password}</value>
- </property>
- -->
-
- <!--If "MsiTokenProvider" is set as key provider, set tenantGuid and client id below-->
- <!--
- <property>
- <name>fs.azure.account.oauth2.msi.tenant.{YOURACCOUNT}.dfs.core.windows.net</name>
- <value>{tenantGuid}</value>
- <description>msi tenantGuid.</description>
- </property>
-
- <property>
- <name>fs.azure.account.oauth2.client.id.{YOURACCOUNT}.dfs.core.windows.net</name>
- <value>{client id}</value>
- <description>AAD client id.</description>
- </property>
- -->
-
- <!--If "RefreshTokenBasedTokenProvider" is set as key provider, set refresh token and client id below-->
- <!--
- <property>
- <name>fs.azure.account.oauth2.refresh.token.{YOURACCOUNT}.dfs.core.windows.net</name>
- <value>{refresh token}</value>
- <description>refresh token.</description>
- </property>
-
- <property>
- <name>fs.azure.account.oauth2.client.id.{YOURACCOUNT}.dfs.core.windows.net</name>
- <value>{client id}</value>
- <description>AAD client id.</description>
- </property>
- -->
-
- <!--Below four configure properties are provided for Blob Data Contributor a
- nd Blob Data Reader OAuth access test "ITestAzureBlobFileSystemOauth",
- using"ClientCredsTokenProvider" as key
- provider. These are for test only.-->
-
- <!--
- <property>
- <name>fs.azure.account.oauth2.contributor.client.id</name>
- <value>{ID of client which has Data Contributor role}</value>
- </property>
-
- <property>
- <name>fs.azure.account.oauth2.contributor.client.secret</name>
- <value>{secret of client which has Data Contributor role}</value>
- </property>
-
- <property>
- <name>fs.azure.account.oauth2.reader.client.id</name>
- <value>{ID of client which has Data Reader role}</value>
- </property>
-
- <property>
- <name>fs.azure.account.oauth2.reader.client.secret</name>
- <value>{Secret of client which has Data Reader role}</value>
- </property>
- -->
-
-
-
- <!-- Save above configuration properties in a separate file named -->
- <!-- azure-bfs-auth-keys.xml in the same directory as this file. -->
- <!-- DO NOT ADD azure-bfs-auth-keys.xml TO REVISION CONTROL. The keys to your -->
- <!-- Azure Storage account are a secret and must not be shared. -->
-
- <include xmlns="http://www.w3.org/2001/XInclude" href="azure-bfs-auth-keys.xml">
- <fallback />
- </include>
-
- <!--<property>-->
- <!--<name>fs.azure.test.host.name</name>-->
- <!--<value>{HOSTNAME}</value>-->
- <!--</property>-->
-
- <!--<property>-->
- <!--<name>fs.azure.test.host.port</name>-->
- <!--<value>{PORT}</value>-->
- <!--</property>-->
-
- <!--<property>-->
- <!--<name>fs.azure.abfs.emulator.enabled</name>-->
- <!--<value>false</value>-->
- <!--</property>-->
-
-</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d58ea7c9/hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml b/hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml
index 8cea256..a36a391 100644
--- a/hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml
+++ b/hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml
@@ -16,49 +16,39 @@
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration xmlns:xi="http://www.w3.org/2001/XInclude">
- <!-- For tests against live azure, provide the following account information -->
+ <!--==================== WASB CONFIGURATION ====================-->
+ <!-- For tests against azure-emulator -->
+ <property>
+ <name>fs.azure.test.emulator</name>
+ <value>false</value>
+ </property>
- <!--
- <property>
- <name>fs.azure.test.account.name</name>
- <value>{ACCOUNTNAME}.blob.core.windows.net</value>
- </property>
- <property>
- <name>fs.azure.account.key.{ACCOUNTNAME}.blob.core.windows.net</name>
- <value>{ACCOUNTKEY}</value>
- </property>
- -->
+ <property>
+ <name>fs.azure.secure.mode</name>
+ <value>false</value>
+ </property>
- <!-- uncomment to test in Azure secure mode -->
- <!--
- <property>
- <name>fs.azure.secure.mode</name>
- <value>true</value>
- </property>
- -->
+ <!--==================== ABFS CONFIGURATION ====================-->
+ <!-- SEE relevant section in "site/markdown/testing_azure.md"-->
+ <!--========== CONFIGURATION SHARED BY WASB AND ABFS ==========-->
<property>
<name>fs.azure.user.agent.prefix</name>
<value>MSFT</value>
</property>
- <!-- Save the above configuration properties in a separate file named -->
- <!-- azure-auth-keys.xml in the same directory as this file. -->
- <!-- DO NOT ADD azure-auth-keys.xml TO REVISION CONTROL. The keys to your -->
- <!-- Azure Storage account are a secret and must not be shared. -->
+ <!--====================== IMPORTANT!! ========================-->
+ <!--WARNING!!
+ PLEASE CREATE A FILE NAMED "azure-auth-keys.xml"
+ AND SEE "site/markdown/testing_azure.md", THEN
+ STORE THE CONFIGURATION PROPERTIES WITHIN IT.
+ TO PREVENT ACCIDENTAL LEAKS OF YOUR STORAGE ACCOUNT CREDENTIALS,
+ THIS FILE IS LISTED IN .gitignore TO PREVENT YOU FROM INCLUDING
+ IT IN PATCHES OR COMMITS. -->
+ <!--=============================================================-->
+
<include xmlns="http://www.w3.org/2001/XInclude" href="azure-auth-keys.xml">
<fallback />
</include>
- <!-- For tests against azure-emulator -->
- <!--
- <property>
- <name>fs.azure.test.emulator</name>
- <value>true</value>
- </property>
- -->
- <property>
- <name>fs.AbstractFileSystem.wasb.impl</name>
- <value>org.apache.hadoop.fs.azure.Wasb</value>
- </property>
</configuration>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org