You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ae...@apache.org on 2017/09/18 22:12:12 UTC
[29/50] [abbrv] hadoop git commit: HADOOP-14553. Add (parallelized)
integration tests to hadoop-azure Contributed by Steve Loughran
HADOOP-14553. Add (parallelized) integration tests to hadoop-azure
Contributed by Steve Loughran
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2d2d97fa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2d2d97fa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2d2d97fa
Branch: refs/heads/HDFS-7240
Commit: 2d2d97fa7d4224369b3c13bc4a45e8cc9e29afb1
Parents: 11390c2
Author: Steve Loughran <st...@apache.org>
Authored: Fri Sep 15 17:03:01 2017 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Sep 15 17:03:01 2017 +0100
----------------------------------------------------------------------
.../hadoop/fs/FileSystemContractBaseTest.java | 11 +-
.../fs/contract/AbstractContractOpenTest.java | 4 +-
.../fs/contract/AbstractContractSeekTest.java | 2 +-
hadoop-tools/hadoop-azure/pom.xml | 251 ++++++
.../fs/azure/AzureNativeFileSystemStore.java | 2 +-
.../hadoop-azure/src/site/markdown/index.md | 94 +-
.../src/site/markdown/testing_azure.md | 576 ++++++++++++
.../hadoop/fs/azure/AbstractWasbTestBase.java | 136 ++-
.../fs/azure/AbstractWasbTestWithTimeout.java | 73 ++
.../fs/azure/AzureBlobStorageTestAccount.java | 42 +-
.../azure/ITestAzureConcurrentOutOfBandIo.java | 179 ++++
...zureConcurrentOutOfBandIoWithSecureMode.java | 33 +
.../ITestAzureFileSystemErrorConditions.java | 243 +++++
.../fs/azure/ITestBlobDataValidation.java | 244 ++++++
.../fs/azure/ITestBlobTypeSpeedDifference.java | 163 ++++
.../fs/azure/ITestBlockBlobInputStream.java | 874 ++++++++++++++++++
.../hadoop/fs/azure/ITestContainerChecks.java | 194 ++++
...estFileSystemOperationExceptionHandling.java | 283 ++++++
...TestFileSystemOperationExceptionMessage.java | 79 ++
...perationsExceptionHandlingMultiThreaded.java | 366 ++++++++
.../ITestFileSystemOperationsWithThreads.java | 821 +++++++++++++++++
...stNativeAzureFSAuthWithBlobSpecificKeys.java | 40 +
.../ITestNativeAzureFSAuthorizationCaching.java | 53 ++
.../azure/ITestNativeAzureFSPageBlobLive.java | 43 +
.../azure/ITestNativeAzureFileSystemAppend.java | 350 ++++++++
...ativeAzureFileSystemAtomicRenameDirList.java | 55 ++
...veAzureFileSystemAuthorizationWithOwner.java | 122 +++
...ITestNativeAzureFileSystemClientLogging.java | 136 +++
...estNativeAzureFileSystemConcurrencyLive.java | 185 ++++
...stNativeAzureFileSystemContractEmulator.java | 65 ++
.../ITestNativeAzureFileSystemContractLive.java | 108 +++
...tiveAzureFileSystemContractPageBlobLive.java | 114 +++
.../azure/ITestNativeAzureFileSystemLive.java | 236 +++++
.../ITestOutOfBandAzureBlobOperationsLive.java | 185 ++++
.../ITestReadAndSeekPageBlobAfterWrite.java | 341 ++++++++
.../fs/azure/ITestWasbRemoteCallHelper.java | 568 ++++++++++++
.../fs/azure/ITestWasbUriAndConfiguration.java | 610 +++++++++++++
.../hadoop/fs/azure/MockWasbAuthorizerImpl.java | 103 ++-
.../fs/azure/NativeAzureFileSystemBaseTest.java | 115 ++-
.../hadoop/fs/azure/RunningLiveWasbTests.txt | 22 -
.../azure/TestAzureConcurrentOutOfBandIo.java | 195 -----
...zureConcurrentOutOfBandIoWithSecureMode.java | 50 --
.../TestAzureFileSystemErrorConditions.java | 244 ------
.../hadoop/fs/azure/TestBlobDataValidation.java | 237 -----
.../hadoop/fs/azure/TestBlobMetadata.java | 7 +-
.../fs/azure/TestBlobOperationDescriptor.java | 3 -
.../fs/azure/TestBlobTypeSpeedDifference.java | 160 ----
.../fs/azure/TestBlockBlobInputStream.java | 875 -------------------
.../fs/azure/TestClientThrottlingAnalyzer.java | 5 +-
.../hadoop/fs/azure/TestContainerChecks.java | 185 ----
...estFileSystemOperationExceptionHandling.java | 269 ------
...TestFileSystemOperationExceptionMessage.java | 79 --
...perationsExceptionHandlingMultiThreaded.java | 330 -------
.../TestFileSystemOperationsWithThreads.java | 821 -----------------
...stNativeAzureFSAuthWithBlobSpecificKeys.java | 44 -
.../TestNativeAzureFSAuthorizationCaching.java | 60 --
.../fs/azure/TestNativeAzureFSPageBlobLive.java | 43 -
.../azure/TestNativeAzureFileSystemAppend.java | 362 --------
...ativeAzureFileSystemAtomicRenameDirList.java | 50 --
.../TestNativeAzureFileSystemAuthorization.java | 53 +-
...veAzureFileSystemAuthorizationWithOwner.java | 122 ---
...TestNativeAzureFileSystemBlockLocations.java | 8 +-
.../TestNativeAzureFileSystemClientLogging.java | 140 ---
.../TestNativeAzureFileSystemConcurrency.java | 29 +-
...estNativeAzureFileSystemConcurrencyLive.java | 184 ----
...stNativeAzureFileSystemContractEmulator.java | 48 -
.../TestNativeAzureFileSystemContractLive.java | 80 --
...TestNativeAzureFileSystemContractMocked.java | 3 +
...tiveAzureFileSystemContractPageBlobLive.java | 93 --
.../TestNativeAzureFileSystemFileNameCheck.java | 28 +-
.../fs/azure/TestNativeAzureFileSystemLive.java | 242 -----
.../azure/TestNativeAzureFileSystemMocked.java | 4 +
.../TestNativeAzureFileSystemUploadLogic.java | 78 +-
.../azure/TestOutOfBandAzureBlobOperations.java | 8 +-
.../TestOutOfBandAzureBlobOperationsLive.java | 203 -----
.../TestReadAndSeekPageBlobAfterWrite.java | 355 --------
.../azure/TestShellDecryptionKeyProvider.java | 15 +-
.../apache/hadoop/fs/azure/TestWasbFsck.java | 9 +-
.../fs/azure/TestWasbRemoteCallHelper.java | 569 ------------
.../fs/azure/TestWasbUriAndConfiguration.java | 617 -------------
.../ITestAzureNativeContractAppend.java | 41 +
.../ITestAzureNativeContractCreate.java | 34 +
.../ITestAzureNativeContractDelete.java | 33 +
.../ITestAzureNativeContractDistCp.java | 47 +
.../ITestAzureNativeContractGetFileStatus.java | 35 +
.../contract/ITestAzureNativeContractMkdir.java | 33 +
.../contract/ITestAzureNativeContractOpen.java | 34 +
.../ITestAzureNativeContractRename.java | 34 +
.../contract/ITestAzureNativeContractSeek.java | 34 +
.../contract/NativeAzureFileSystemContract.java | 19 +-
.../contract/TestAzureNativeContractAppend.java | 37 -
.../contract/TestAzureNativeContractCreate.java | 30 -
.../contract/TestAzureNativeContractDelete.java | 30 -
.../contract/TestAzureNativeContractDistCp.java | 33 -
.../TestAzureNativeContractGetFileStatus.java | 30 -
.../contract/TestAzureNativeContractMkdir.java | 30 -
.../contract/TestAzureNativeContractOpen.java | 30 -
.../contract/TestAzureNativeContractRename.java | 30 -
.../contract/TestAzureNativeContractSeek.java | 30 -
.../integration/AbstractAzureScaleTest.java | 66 ++
.../azure/integration/AzureTestConstants.java | 180 ++++
.../fs/azure/integration/AzureTestUtils.java | 479 ++++++++++
.../integration/CleanupTestContainers.java | 87 ++
.../azure/integration/ITestAzureHugeFiles.java | 456 ++++++++++
.../hadoop/fs/azure/integration/Sizes.java | 43 +
.../ITestAzureFileSystemInstrumentation.java | 586 +++++++++++++
.../TestAzureFileSystemInstrumentation.java | 579 ------------
107 files changed, 10227 insertions(+), 7901 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java
index b49dd53..a4ccee3 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java
@@ -61,7 +61,16 @@ public abstract class FileSystemContractBaseTest {
protected byte[] data = dataset(getBlockSize() * 2, 0, 255);
@Rule
- public Timeout globalTimeout = new Timeout(30000);
+ public Timeout globalTimeout = new Timeout(getGlobalTimeout());
+
+ /**
+ * Get the timeout in milliseconds for each test case.
+ * @return a time in milliseconds.
+ */
+ protected int getGlobalTimeout() {
+ return 30 * 1000;
+ }
+
@Rule
public ExpectedException thrown = ExpectedException.none();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractOpenTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractOpenTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractOpenTest.java
index f9b16f4..ccf188f 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractOpenTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractOpenTest.java
@@ -122,7 +122,7 @@ public abstract class AbstractContractOpenTest extends AbstractFSContractTestBas
Path path = path("testopenfiletwice.txt");
byte[] block = dataset(TEST_FILE_LEN, 0, 255);
//this file now has a simple rule: offset => value
- createFile(getFileSystem(), path, false, block);
+ createFile(getFileSystem(), path, true, block);
//open first
FSDataInputStream instream1 = getFileSystem().open(path);
FSDataInputStream instream2 = null;
@@ -150,7 +150,7 @@ public abstract class AbstractContractOpenTest extends AbstractFSContractTestBas
int base = 0x40; // 64
byte[] block = dataset(len, base, base + len);
//this file now has a simple rule: offset => (value | 0x40)
- createFile(getFileSystem(), path, false, block);
+ createFile(getFileSystem(), path, true, block);
//open first
instream = getFileSystem().open(path);
assertEquals(base, instream.read());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractSeekTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractSeekTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractSeekTest.java
index 3e71682..7af3cb0 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractSeekTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractSeekTest.java
@@ -341,7 +341,7 @@ public abstract class AbstractContractSeekTest extends AbstractFSContractTestBas
int filesize = 10 * 1024;
byte[] buf = dataset(filesize, 0, 255);
Path randomSeekFile = path("testrandomseeks.bin");
- createFile(getFileSystem(), randomSeekFile, false, buf);
+ createFile(getFileSystem(), randomSeekFile, true, buf);
Random r = new Random();
// Record the sequence of seeks and reads which trigger a failure.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/pom.xml b/hadoop-tools/hadoop-azure/pom.xml
index 0c5ac63..b479872 100644
--- a/hadoop-tools/hadoop-azure/pom.xml
+++ b/hadoop-tools/hadoop-azure/pom.xml
@@ -34,6 +34,15 @@
<properties>
<file.encoding>UTF-8</file.encoding>
<downloadSources>true</downloadSources>
+ <hadoop.tmp.dir>${project.build.directory}/test</hadoop.tmp.dir>
+ <!-- are scale tests enabled ? -->
+ <fs.azure.scale.test.enabled>unset</fs.azure.scale.test.enabled>
+ <!-- Size in MB of huge files. -->
+ <fs.azure.scale.test.huge.filesize>unset</fs.azure.scale.test.huge.filesize>
+ <!-- Size in MB of the partion size in huge file uploads. -->
+ <fs.azure.scale.test.huge.partitionsize>unset</fs.azure.scale.test.huge.partitionsize>
+ <!-- Timeout in seconds for scale tests.-->
+ <fs.azure.scale.test.timeout>7200</fs.azure.scale.test.timeout>
</properties>
<build>
@@ -224,4 +233,246 @@
</dependency>
</dependencies>
+
+ <profiles>
+ <profile>
+ <id>parallel-tests</id>
+ <activation>
+ <property>
+ <name>parallel-tests</name>
+ </property>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>create-parallel-tests-dirs</id>
+ <phase>test-compile</phase>
+ <configuration>
+ <target>
+ <script language="javascript"><![CDATA[
+ var baseDirs = [
+ project.getProperty("test.build.data"),
+ project.getProperty("test.build.dir"),
+ project.getProperty("hadoop.tmp.dir")
+ ];
+ for (var i in baseDirs) {
+ for (var j = 1; j <= ${testsThreadCount}; ++j) {
+ var mkdir = project.createTask("mkdir");
+ mkdir.setDir(new java.io.File(baseDirs[i], j));
+ mkdir.perform();
+ }
+ }
+ ]]></script>
+ </target>
+ </configuration>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>default-test</id>
+ <goals>
+ <goal>test</goal>
+ </goals>
+ <configuration>
+ <forkCount>1</forkCount>
+ <forkCount>${testsThreadCount}</forkCount>
+ <reuseForks>false</reuseForks>
+ <argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine>
+ <forkedProcessTimeoutInSeconds>${fs.azure.scale.test.timeout}</forkedProcessTimeoutInSeconds>
+ <systemPropertyVariables>
+ <test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data>
+ <test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir>
+ <hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir>
+ <test.unique.fork.id>fork-${surefire.forkNumber}</test.unique.fork.id>
+ <fs.azure.scale.test.enabled>${fs.azure.scale.test.enabled}</fs.azure.scale.test.enabled>
+ <fs.azure.scale.test.huge.filesize>${fs.azure.scale.test.huge.filesize}</fs.azure.scale.test.huge.filesize>
+ <fs.azure.scale.test.huge.huge.partitionsize>${fs.azure.scale.test.huge.partitionsize}</fs.azure.scale.test.huge.huge.partitionsize>
+ <fs.azure.scale.test.timeout>${fs.azure.scale.test.timeout}</fs.azure.scale.test.timeout>
+ </systemPropertyVariables>
+ <includes>
+ <include>**/Test*.java</include>
+ </includes>
+ <excludes>
+ <exclude>**/TestRollingWindowAverage*.java</exclude>
+ </excludes>
+ </configuration>
+ </execution>
+ <execution>
+ <id>serialized-test</id>
+ <goals>
+ <goal>test</goal>
+ </goals>
+ <configuration>
+ <forkCount>1</forkCount>
+ <reuseForks>false</reuseForks>
+ <argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine>
+ <forkedProcessTimeoutInSeconds>${fs.azure.scale.test.timeout}</forkedProcessTimeoutInSeconds>
+ <systemPropertyVariables>
+ <test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data>
+ <test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir>
+ <hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir>
+ <test.unique.fork.id>fork-${surefire.forkNumber}</test.unique.fork.id>
+ <fs.azure.scale.test.enabled>${fs.azure.scale.test.enabled}</fs.azure.scale.test.enabled>
+ <fs.azure.scale.test.huge.filesize>${fs.azure.scale.test.huge.filesize}</fs.azure.scale.test.huge.filesize>
+ <fs.azure.scale.test.huge.huge.partitionsize>${fs.azure.scale.test.huge.partitionsize}</fs.azure.scale.test.huge.huge.partitionsize>
+ <fs.azure.scale.test.timeout>${fs.azure.scale.test.timeout}</fs.azure.scale.test.timeout>
+ </systemPropertyVariables>
+ <includes>
+ <include>**/TestRollingWindowAverage*.java</include>
+ </includes>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-failsafe-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>default-integration-test</id>
+ <goals>
+ <goal>integration-test</goal>
+ <goal>verify</goal>
+ </goals>
+ <configuration>
+ <forkCount>${testsThreadCount}</forkCount>
+ <reuseForks>false</reuseForks>
+ <argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine>
+ <forkedProcessTimeoutInSeconds>${fs.azure.scale.test.timeout}</forkedProcessTimeoutInSeconds>
+ <systemPropertyVariables>
+ <!-- Tell tests that they are being executed in parallel -->
+ <test.parallel.execution>true</test.parallel.execution>
+ <test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data>
+ <test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir>
+ <hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir>
+
+ <!-- Due to a Maven quirk, setting this to just -->
+ <!-- surefire.forkNumber won't do the parameter -->
+ <!-- substitution. Putting a prefix in front of it like -->
+ <!-- "fork-" makes it work. -->
+ <test.unique.fork.id>fork-${surefire.forkNumber}</test.unique.fork.id>
+ <!-- Propagate scale parameters -->
+ <fs.azure.scale.test.enabled>${fs.azure.scale.test.enabled}</fs.azure.scale.test.enabled>
+ <fs.azure.scale.test.huge.filesize>${fs.azure.scale.test.huge.filesize}</fs.azure.scale.test.huge.filesize>
+ <fs.azure.scale.test.huge.huge.partitionsize>${fs.azure.scale.test.huge.partitionsize}</fs.azure.scale.test.huge.huge.partitionsize>
+ <fs.azure.scale.test.timeout>${fs.azure.scale.test.timeout}</fs.azure.scale.test.timeout>
+ </systemPropertyVariables>
+ <!-- Some tests cannot run in parallel. Tests that cover -->
+ <!-- access to the root directory must run in isolation -->
+ <!-- from anything else that could modify the bucket. -->
+ <!-- azure tests that cover multi-part upload must run in -->
+ <!-- isolation, because the file system is configured to -->
+ <!-- purge existing multi-part upload data on -->
+ <!-- initialization. MiniYARNCluster has not yet been -->
+ <!-- changed to handle parallel test execution gracefully. -->
+ <!-- Exclude all of these tests from parallel execution, -->
+ <!-- and instead run them sequentially in a separate -->
+ <!-- Surefire execution step later. -->
+ <includes>
+ <include>**/ITest*.java</include>
+ </includes>
+ <excludes>
+ <exclude>**/ITestFileSystemOperationsExceptionHandlingMultiThreaded.java</exclude>
+ <exclude>**/ITestFileSystemOperationsWithThreads.java</exclude>
+ <exclude>**/ITestOutOfBandAzureBlobOperationsLive.java</exclude>
+ <exclude>**/ITestNativeAzureFileSystemAuthorizationWithOwner.java</exclude>
+ <exclude>**/ITestNativeAzureFileSystemConcurrencyLive.java</exclude>
+ <exclude>**/ITestNativeAzureFileSystemLive.java</exclude>
+ <exclude>**/ITestNativeAzureFSPageBlobLive.java</exclude>
+ <exclude>**/ITestWasbRemoteCallHelper.java</exclude>
+ <exclude>**/ITestBlockBlobInputStream.java</exclude>
+ </excludes>
+ </configuration>
+ </execution>
+ <!-- Do a sequential run for tests that cannot handle -->
+ <!-- parallel execution. -->
+ <execution>
+ <id>sequential-integration-tests</id>
+ <goals>
+ <goal>integration-test</goal>
+ <goal>verify</goal>
+ </goals>
+ <configuration>
+ <forkedProcessTimeoutInSeconds>${fs.azure.scale.test.timeout}</forkedProcessTimeoutInSeconds>
+ <systemPropertyVariables>
+ <test.parallel.execution>false</test.parallel.execution>
+ <fs.azure.scale.test.enabled>${fs.azure.scale.test.enabled}</fs.azure.scale.test.enabled>
+ <fs.azure.scale.test.huge.filesize>${fs.azure.scale.test.huge.filesize}</fs.azure.scale.test.huge.filesize>
+ <fs.azure.scale.test.huge.huge.partitionsize>${fs.azure.scale.test.huge.partitionsize}</fs.azure.scale.test.huge.huge.partitionsize>
+ <fs.azure.scale.test.timeout>${fs.azure.scale.test.timeout}</fs.azure.scale.test.timeout>
+ </systemPropertyVariables>
+ <includes>
+ <include>**/ITestFileSystemOperationsExceptionHandlingMultiThreaded.java</include>
+ <include>**/ITestFileSystemOperationsWithThreads.java</include>
+ <include>**/ITestOutOfBandAzureBlobOperationsLive.java</include>
+ <include>**/ITestNativeAzureFileSystemAuthorizationWithOwner.java</include>
+ <include>**/ITestNativeAzureFileSystemConcurrencyLive.java</include>
+ <include>**/ITestNativeAzureFileSystemLive.java</include>
+ <include>**/ITestWasbRemoteCallHelper.java</include>
+ <include>**/ITestBlockBlobInputStream.java</include>
+ </includes>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ <profile>
+ <id>sequential-tests</id>
+ <activation>
+ <property>
+ <name>!parallel-tests</name>
+ </property>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-failsafe-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>integration-test</goal>
+ <goal>verify</goal>
+ </goals>
+ <configuration>
+ <systemPropertyVariables>
+ <!-- Propagate scale parameters -->
+ <fs.azure.scale.test.enabled>${fs.azure.scale.test.enabled}</fs.azure.scale.test.enabled>
+ <fs.azure.scale.test.huge.filesize>${fs.azure.scale.test.huge.filesize}</fs.azure.scale.test.huge.filesize>
+ <fs.azure.scale.test.timeout>${fs.azure.scale.test.timeout}</fs.azure.scale.test.timeout>
+ </systemPropertyVariables>
+ <forkedProcessTimeoutInSeconds>${fs.azure.scale.test.timeout}</forkedProcessTimeoutInSeconds>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+
+ <!-- Turn on scale tests-->
+ <profile>
+ <id>scale</id>
+ <activation>
+ <property>
+ <name>scale</name>
+ </property>
+ </activation>
+ <properties>
+ <fs.azure.scale.test.enabled>true</fs.azure.scale.test.enabled>
+ </properties>
+ </profile>
+ </profiles>
</project>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
index 639862f..f1031b4 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
@@ -346,7 +346,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
private String delegationToken;
/** The error message template when container is not accessible. */
- static final String NO_ACCESS_TO_CONTAINER_MSG = "No credentials found for "
+ public static final String NO_ACCESS_TO_CONTAINER_MSG = "No credentials found for "
+ "account %s in the configuration, and its container %s is not "
+ "accessible using anonymous credentials. Please check if the container "
+ "exists first. If it is not publicly available, you have to provide "
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/site/markdown/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/index.md b/hadoop-tools/hadoop-azure/src/site/markdown/index.md
index 466bf0b..876d7cc 100644
--- a/hadoop-tools/hadoop-azure/src/site/markdown/index.md
+++ b/hadoop-tools/hadoop-azure/src/site/markdown/index.md
@@ -519,96 +519,8 @@ The maximum number of entries that that cache can hold can be customized using t
<value>true</value>
</property>
```
-## Testing the hadoop-azure Module
-The hadoop-azure module includes a full suite of unit tests. Most of the tests
-will run without additional configuration by running `mvn test`. This includes
-tests against mocked storage, which is an in-memory emulation of Azure Storage.
-
-A selection of tests can run against the
-[Azure Storage Emulator](http://msdn.microsoft.com/en-us/library/azure/hh403989.aspx)
-which is a high-fidelity emulation of live Azure Storage. The emulator is
-sufficient for high-confidence testing. The emulator is a Windows executable
-that runs on a local machine.
-
-To use the emulator, install Azure SDK 2.3 and start the storage emulator. Then,
-edit `src/test/resources/azure-test.xml` and add the following property:
-
-```xml
-<property>
- <name>fs.azure.test.emulator</name>
- <value>true</value>
-</property>
-```
-
-There is a known issue when running tests with the emulator. You may see the
-following failure message:
-
- com.microsoft.windowsazure.storage.StorageException: The value for one of the HTTP headers is not in the correct format.
-
-To resolve this, restart the Azure Emulator. Ensure it v3.2 or later.
-
-It's also possible to run tests against a live Azure Storage account by saving a
-file to `src/test/resources/azure-auth-keys.xml` and setting
-the name of the storage account and its access key.
-
-For example:
-
-```xml
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<configuration>
- <property>
- <name>fs.azure.test.account.name</name>
- <value>{ACCOUNTNAME}.blob.core.windows.net</value>
- </property>
- <property>
- <name>fs.azure.account.key.{ACCOUNTNAME}.blob.core.windows.net</name>
- <value>{ACCOUNT ACCESS KEY}</value>
- </property>
-</configuration>
-```
-
-To run contract tests, set the WASB file system URI in `src/test/resources/azure-auth-keys.xml`
-and the account access key. For example:
-
-```xml
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<configuration>
- <property>
- <name>fs.contract.test.fs.wasb</name>
- <value>wasb://{CONTAINERNAME}@{ACCOUNTNAME}.blob.core.windows.net</value>
- <description>The name of the azure file system for testing.</description>
- </property>
- <property>
- <name>fs.azure.account.key.{ACCOUNTNAME}.blob.core.windows.net</name>
- <value>{ACCOUNT ACCESS KEY}</value>
- </property>
-</configuration>
-```
-
-Overall, to run all the tests using `mvn test`, a sample `azure-auth-keys.xml` is like following:
-
-```xml
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<configuration>
- <property>
- <name>fs.azure.test.account.name</name>
- <value>{ACCOUNTNAME}.blob.core.windows.net</value>
- </property>
- <property>
- <name>fs.azure.account.key.{ACCOUNTNAME}.blob.core.windows.net</name>
- <value>{ACCOUNT ACCESS KEY}</value>
- </property>
- <property>
- <name>fs.contract.test.fs.wasb</name>
- <value>wasb://{CONTAINERNAME}@{ACCOUNTNAME}.blob.core.windows.net</value>
- </property>
-</configuration>
-```
-
-DO NOT ADD `azure-auth-keys.xml` TO REVISION CONTROL. The keys to your Azure
-Storage account are a secret and must not be shared.
+## Further Reading
+* [Testing the Azure WASB client](testing_azure.html).
+* MSDN article, [Understanding Block Blobs, Append Blobs, and Page Blobs](https://docs.microsoft.com/en-us/rest/api/storageservices/understanding-block-blobs--append-blobs--and-page-blobs)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/site/markdown/testing_azure.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/testing_azure.md b/hadoop-tools/hadoop-azure/src/site/markdown/testing_azure.md
new file mode 100644
index 0000000..b58e68b
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/site/markdown/testing_azure.md
@@ -0,0 +1,576 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# Testing the Azure WASB client
+
+<!-- MACRO{toc|fromDepth=0|toDepth=5} -->
+
+This module includes both unit tests, which can run in isolation without
+connecting to the Azure Storage service, and integration tests, which require a working
+connection to interact with a container. Unit test suites follow the naming
+convention `Test*.java`. Integration tests follow the naming convention
+`ITest*.java`.
+
+## Policy for submitting patches which affect the `hadoop-azure` module.
+
+The Apache Jenkins infrastucture does not run any cloud integration tests,
+due to the need to keep credentials secure.
+
+### The submitter of any patch is required to run all the integration tests and declare which Azure region they used.
+
+This is important: **patches which do not include this declaration will be ignored**
+
+This policy has proven to be the only mechanism to guarantee full regression
+testing of code changes. Why the declaration of region? Two reasons
+
+1. It helps us identify regressions which only surface against specific endpoints.
+1. It forces the submitters to be more honest about their testing. It's easy
+to lie, "yes, I tested this". To say "yes, I tested this against Azure US-west"
+is a more specific lie and harder to make. And, if you get caught out: you
+lose all credibility with the project.
+
+You don't need to test from a VM within the Azure infrastructure, all you need
+are credentials.
+
+It's neither hard nor expensive to run the tests; if you can't,
+there's no guarantee your patch works. The reviewers have enough to do, and
+don't have the time to do these tests, especially as every failure will simply
+make for a slow iterative development.
+
+Please: run the tests. And if you don't, we are sorry for declining your
+patch, but we have to.
+
+
+### What if there's an intermittent failure of a test?
+
+Some of the tests do fail intermittently, especially in parallel runs.
+If this happens, try to run the test on its own to see if the test succeeds.
+
+If it still fails, include this fact in your declaration. We know some tests
+are intermittently unreliable.
+
+### What if the tests are timing out or failing over my network connection?
+
+The tests are designed to be configurable for different
+timeouts. If you are seeing problems and this configuration isn't working,
+that's a sign of the configuration mechanism isn't complete. If it's happening
+in the production code, that could be a sign of a problem which may surface
+over long-haul connections. Please help us identify and fix these problems
+— especially as you are the one best placed to verify the fixes work.
+
+## Setting up the tests
+
+## Testing the `hadoop-azure` Module
+
+The `hadoop-azure` module includes a full suite of unit tests. Many of the tests
+will run without additional configuration by running `mvn test`. This includes
+tests against mocked storage, which is an in-memory emulation of Azure Storage.
+
+The integration tests are designed to test directly against an Azure storage
+service, and require an account and credentials in order to run.
+
+This is done by creating the file to `src/test/resources/azure-auth-keys.xml`
+and setting the name of the storage account and its access key.
+
+For example:
+
+```xml
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<configuration>
+ <property>
+ <name>fs.azure.test.account.name</name>
+ <value>{ACCOUNTNAME}.blob.core.windows.net</value>
+ </property>
+ <property>
+ <name>fs.azure.account.key.{ACCOUNTNAME}.blob.core.windows.net</name>
+ <value>{ACCOUNT ACCESS KEY}</value>
+ </property>
+</configuration>
+```
+
+To run contract tests, set the WASB file system URI in `src/test/resources/azure-auth-keys.xml`
+and the account access key. For example:
+
+```xml
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<configuration>
+ <property>
+ <name>fs.contract.test.fs.wasb</name>
+ <value>wasb://{CONTAINERNAME}@{ACCOUNTNAME}.blob.core.windows.net</value>
+ <description>The name of the azure file system for testing.</description>
+ </property>
+ <property>
+ <name>fs.azure.account.key.{ACCOUNTNAME}.blob.core.windows.net</name>
+ <value>{ACCOUNT ACCESS KEY}</value>
+ </property>
+</configuration>
+```
+
+Overall, to run all the tests using `mvn test`, a sample `azure-auth-keys.xml` is like following:
+
+```xml
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<configuration>
+ <property>
+ <name>fs.azure.test.account.name</name>
+ <value>{ACCOUNTNAME}.blob.core.windows.net</value>
+ </property>
+ <property>
+ <name>fs.azure.account.key.{ACCOUNTNAME}.blob.core.windows.net</name>
+ <value>{ACCOUNT ACCESS KEY}</value>
+ </property>
+ <property>
+ <name>fs.contract.test.fs.wasb</name>
+ <value>wasb://{CONTAINERNAME}@{ACCOUNTNAME}.blob.core.windows.net</value>
+ </property>
+</configuration>
+```
+
+DO NOT ADD `azure-auth-keys.xml` TO REVISION CONTROL. The keys to your Azure
+Storage account are a secret and must not be shared.
+
+
+## Running the Tests
+
+After completing the configuration, execute the test run through Maven.
+
+```bash
+mvn -T 1C clean verify
+```
+
+It's also possible to execute multiple test suites in parallel by passing the
+`parallel-tests` property on the command line. The tests spend most of their
+time blocked on network I/O, so running in parallel tends to
+complete full test runs faster.
+
+```bash
+mvn -T 1C -Dparallel-tests clean verify
+```
+
+Some tests must run with exclusive access to the storage container, so even with the
+`parallel-tests` property, several test suites will run in serial in a separate
+Maven execution step after the parallel tests.
+
+By default, `parallel-tests` runs 4 test suites concurrently. This can be tuned
+by passing the `testsThreadCount` property.
+
+```bash
+mvn -T 1C -Dparallel-tests -DtestsThreadCount=8 clean verify
+```
+
+<!---
+To run just unit tests, which do not require Azure connectivity or credentials,
+use any of the above invocations, but switch the goal to `test` instead of
+`verify`.
+-->
+
+```bash
+mvn -T 1C clean test
+
+mvn -T 1C -Dparallel-tests clean test
+
+mvn -T 1C -Dparallel-tests -DtestsThreadCount=8 clean test
+```
+
+To run only a specific named subset of tests, pass the `test` property for unit
+tests or the `it.test` property for integration tests.
+
+```bash
+mvn -T 1C clean test -Dtest=TestRollingWindowAverage
+
+mvn -T 1C clean verify -Dscale -Dit.test=ITestFileSystemOperationExceptionMessage -Dtest=none
+
+mvn -T 1C clean verify -Dtest=none -Dit.test=ITest*
+
+```
+
+Note
+
+1. When running a specific subset of tests, the patterns passed in `test`
+and `it.test` override the configuration of which tests need to run in isolation
+in a separate serial phase (mentioned above). This can cause unpredictable
+results, so the recommendation is to avoid passing `parallel-tests` in
+combination with `test` or `it.test`. If you know that you are specifying only
+tests that can run safely in parallel, then it will work. For wide patterns,
+like `ITest*` shown above, it may cause unpredictable test failures.
+
+2. The command line shell may try to expand the "*" and sometimes the "#" symbols
+in test patterns. In such situations, escape the character it with a "\\" prefix.
+Example:
+
+ mvn -T 1C clean verify -Dtest=none -Dit.test=ITest\*
+
+
+## Viewing the results
+
+Integration test results and logs are stored in `target/failsafe-reports/`.
+An HTML report can be generated during site generation, or with the `surefire-report`
+plugin:
+
+```bash
+
+# for the unit tests
+mvn -T 1C surefire-report:report-only
+
+# for the integration tests
+mvn -T 1C surefire-report:failsafe-report-only
+
+# all reports for this module
+mvn -T 1C site:site
+```
+
+## Scale Tests
+
+There are a set of tests designed to measure the scalability and performance
+at scale of the filesystem client, *Scale Tests*. Tests include: creating
+and traversing directory trees, uploading large files, renaming them,
+deleting them, seeking through the files, performing random IO, and others.
+This makes them a foundational part of the benchmarking.
+
+By their very nature they are slow. And, as their execution time is often
+limited by bandwidth between the computer running the tests and the Azure endpoint,
+parallel execution does not speed these tests up.
+
+### Enabling the Scale Tests
+
+The tests are enabled if the `scale` property is set in the maven build
+this can be done regardless of whether or not the parallel test profile
+is used
+
+```bash
+mvn -T 1C verify -Dscale
+
+mvn -T 1C verify -Dparallel-tests -Dscale -DtestsThreadCount=8
+```
+
+The most bandwidth intensive tests (those which upload data) always run
+sequentially; those which are slow due to HTTPS setup costs or server-side
+actions are included in the set of parallelized tests.
+
+
+### Scale test tuning options
+
+
+Some of the tests can be tuned from the maven build or from the
+configuration file used to run the tests.
+
+```bash
+mvn -T 1C verify -Dparallel-tests -Dscale -DtestsThreadCount=8 -Dfs.azure.scale.test.huge.filesize=128M
+```
+
+The algorithm is
+
+1. The value is queried from the configuration file, using a default value if
+it is not set.
+1. The value is queried from the JVM System Properties, where it is passed
+down by maven.
+1. If the system property is null, an empty string, or it has the value `unset`,
+then the configuration value is used. The `unset` option is used to
+[work round a quirk in maven property propagation](http://stackoverflow.com/questions/7773134/null-versus-empty-arguments-in-maven).
+
+Only a few properties can be set this way; more will be added.
+
+| Property | Meaninging |
+|-----------|-------------|
+| `fs.azure.scale.test.huge.filesize`| Size for huge file uploads |
+| `fs.azure.scale.test.huge.huge.partitionsize`| Size for partitions in huge file uploads |
+
+The file and partition sizes are numeric values with a k/m/g/t/p suffix depending
+on the desired size. For example: 128M, 128m, 2G, 2G, 4T or even 1P.
+
+#### Scale test configuration options
+
+Some scale tests perform multiple operations (such as creating many directories).
+
+The exact number of operations to perform is configurable in the option
+`scale.test.operation.count`
+
+```xml
+<property>
+ <name>scale.test.operation.count</name>
+ <value>10</value>
+</property>
+```
+
+Larger values generate more load, and are recommended when testing locally,
+or in batch runs.
+
+Smaller values results in faster test runs, especially when the object
+store is a long way away.
+
+Operations which work on directories have a separate option: this controls
+the width and depth of tests creating recursive directories. Larger
+values create exponentially more directories, with consequent performance
+impact.
+
+```xml
+<property>
+ <name>scale.test.directory.count</name>
+ <value>2</value>
+</property>
+```
+
+DistCp tests targeting Azure support a configurable file size. The default is
+10 MB, but the configuration value is expressed in KB so that it can be tuned
+smaller to achieve faster test runs.
+
+```xml
+<property>
+ <name>scale.test.distcp.file.size.kb</name>
+ <value>10240</value>
+</property>
+```
+
+Azure-specific scale test properties are
+
+##### `fs.azure.scale.test.huge.filesize`: size in MB for "Huge file tests".
+
+The Huge File tests validate Azure storages's ability to handle large files —the property
+`fs.azure.scale.test.huge.filesize` declares the file size to use.
+
+```xml
+<property>
+ <name>fs.azure.scale.test.huge.filesize</name>
+ <value>200M</value>
+</property>
+```
+
+Tests at this scale are slow: they are best executed from hosts running in
+the cloud infrastructure where the storage endpoint is based.
+
+## Using the emulator
+
+A selection of tests can run against the
+[Azure Storage Emulator](http://msdn.microsoft.com/en-us/library/azure/hh403989.aspx)
+which is a high-fidelity emulation of live Azure Storage. The emulator is
+sufficient for high-confidence testing. The emulator is a Windows executable
+that runs on a local machine.
+
+To use the emulator, install Azure SDK 2.3 and start the storage emulator. Then,
+edit `src/test/resources/azure-test.xml` and add the following property:
+
+```xml
+<property>
+ <name>fs.azure.test.emulator</name>
+ <value>true</value>
+</property>
+```
+
+There is a known issue when running tests with the emulator. You may see the
+following failure message:
+
+ com.microsoft.windowsazure.storage.StorageException: The value for one of the HTTP headers is not in the correct format.
+
+To resolve this, restart the Azure Emulator. Ensure it is v3.2 or later.
+
+
+## Debugging Test failures
+
+Logging at debug level is the standard way to provide more diagnostics output;
+after setting this rerun the tests
+
+```properties
+log4j.logger.org.apache.hadoop.fs.azure=DEBUG
+```
+
+## Adding new tests
+
+New tests are always welcome. Bear in mind that we need to keep costs
+and test time down, which is done by
+
+* Not duplicating tests.
+* Being efficient in your use of Hadoop API calls.
+* Isolating large/slow tests into the "scale" test group.
+* Designing all tests to execute in parallel (where possible).
+* Adding new probes and predicates into existing tests, albeit carefully.
+
+*No duplication*: if an operation is tested elsewhere, don't repeat it. This
+applies as much for metadata operations as it does for bulk IO. If a new
+test case is added which completely obsoletes an existing test, it is OK
+to cut the previous one —after showing that coverage is not worsened.
+
+*Efficient*: prefer the `getFileStatus()` and examining the results, rather than
+call to `exists()`, `isFile()`, etc.
+
+*Fail with useful information:* provide as much diagnostics as possible
+on a failure. Using `org.apache.hadoop.fs.contract.ContractTestUtils` to make
+assertions about the state of a filesystem helps here.
+
+*Isolating Scale tests*. Any test doing large amounts of IO MUST extend the
+class `AbstractAzureScaleTest`, so only running if `scale` is defined on a build,
+supporting test timeouts configurable by the user. Scale tests should also
+support configurability as to the actual size of objects/number of operations,
+so that behavior at different scale can be verified.
+
+*Designed for parallel execution*. A key need here is for each test suite to work
+on isolated parts of the filesystem. Subclasses of `AbstractWasbTestBase`
+SHOULD use the `path()`, `methodpath()` and `blobpath()` methods,
+to build isolated paths. Tests MUST NOT assume that they have exclusive access
+to a bucket.
+
+*Extending existing tests where appropriate*. This recommendation goes
+against normal testing best practise of "test one thing per method".
+Because it is so slow to create directory trees or upload large files, we do
+not have that luxury. All the tests against real endpoints are integration
+tests where sharing test setup and teardown saves time and money.
+
+A standard way to do this is to extend existing tests with some extra predicates,
+rather than write new tests. When doing this, make sure that the new predicates
+fail with meaningful diagnostics, so any new problems can be easily debugged
+from test logs.
+
+
+### Requirements of new Tests
+
+
+This is what we expect from new tests; they're an extension of the normal
+Hadoop requirements, based on the need to work with remote servers whose
+use requires the presence of secret credentials, where tests may be slow,
+and where finding out why something failed from nothing but the test output
+is critical.
+
+#### Subclasses Existing Shared Base Blasses
+
+There are a set of base classes which should be extended for Azure tests and
+integration tests.
+
+##### `org.apache.hadoop.fs.azure.AbstractWasbTestWithTimeout`
+
+This extends the junit `Assert` class with thread names and timeouts,
+the default timeout being set in `AzureTestConstants.AZURE_TEST_TIMEOUT` to
+ten minutes. The thread names are set to aid analyzing the stack trace of
+a test: a `jstack` call can be used to
+
+##### `org.apache.hadoop.fs.azure.AbstractWasbTestBase`
+
+The base class for tests which use `AzureBlobStorageTestAccount` to create
+mock or live Azure clients; in test teardown it tries to clean up store state.
+
+1. This class requires subclasses to implement `createTestAccount()` to create
+a mock or real test account.
+
+1. The configuration used to create a test account *should* be that from
+`createConfiguration()`; this can be extended in subclasses to tune the settings.
+
+
+##### `org.apache.hadoop.fs.azure.integration.AbstractAzureScaleTest`
+
+This extends `AbstractWasbTestBase` for scale tests; those test which
+only run when `-Dscale` is used to select the "scale" profile.
+These tests have a timeout of 30 minutes, so as to support slow test runs.
+
+Having shared base classes help reduces future maintenance. Please
+use them.
+
+#### Secure
+
+Don't ever log credentials. The credential tests go out of their way to
+not provide meaningful logs or assertion messages precisely to avoid this.
+
+#### Efficient of Time and Money
+
+This means efficient in test setup/teardown, and, ideally, making use of
+existing public datasets to save setup time and tester cost.
+
+
+The reference example is `ITestAzureHugeFiles`:. This marks the test suite as
+`@FixMethodOrder(MethodSorters.NAME_ASCENDING)` then orders the test cases such
+that each test case expects the previous test to have completed (here: uploaded a file,
+renamed a file, ...). This provides for independent tests in the reports, yet still
+permits an ordered sequence of operations. Do note the use of `Assume.assume()`
+to detect when the preconditions for a single test case are not met, hence,
+the tests become skipped, rather than fail with a trace which is really a false alarm.
+
+
+### Works Over Long-haul Links
+
+As well as making file size and operation counts scaleable, this includes
+making test timeouts adequate. The Scale tests make this configurable; it's
+hard coded to ten minutes in `AbstractAzureIntegrationTest()`; subclasses can
+change this by overriding `getTestTimeoutMillis()`.
+
+Equally importantly: support proxies, as some testers need them.
+
+
+### Provides Diagnostics and timing information
+
+1. Create logs, log things.
+1. you can use `AbstractWasbTestBase.describe(format-string, args)` here; it
+adds some newlines so as to be easier to spot.
+1. Use `ContractTestUtils.NanoTimer` to measure the duration of operations,
+and log the output.
+
+#### Fails Meaningfully
+
+The `ContractTestUtils` class contains a whole set of assertions for making
+statements about the expected state of a filesystem, e.g.
+`assertPathExists(FS, path)`, `assertPathDoesNotExists(FS, path)`, and others.
+These do their best to provide meaningful diagnostics on failures (e.g. directory
+listings, file status, ...), so help make failures easier to understand.
+
+At the very least, *do not use `assertTrue()` or `assertFalse()` without
+including error messages*.
+
+
+### Cleans Up Afterwards
+
+Keeps costs down.
+
+1. Do not only cleanup if a test case completes successfully; test suite
+teardown must do it.
+1. That teardown code must check for the filesystem and other fields being
+null before the cleanup. Why? If test setup fails, the teardown methods still
+get called.
+
+### Works Reliably
+
+We really appreciate this — you will too.
+
+
+## Tips
+
+### How to keep your credentials really safe
+
+Although the `auth-keys.xml` file is marged as ignored in git and subversion,
+it is still in your source tree, and there's always that risk that it may
+creep out.
+
+You can avoid this by keeping your keys outside the source tree and
+using an absolute XInclude reference to it.
+
+```xml
+<configuration>
+
+ <include xmlns="http://www.w3.org/2001/XInclude"
+ href="file:///users/qe/.auth-keys.xml" />
+
+</configuration>
+```
+
+### Cleaning up Containers
+
+The Azure tests create containers with the prefix `"wasbtests-"` and delete
+them after the test runs. If a test run is interrupted, these containers
+may not get deleted. There is a special test case which can be manually invoked
+to list and delete these, `CleanupTestContainers`
+
+```bash
+mvn test -Dtest=CleanupTestContainers
+```
+
+This will delete the containers; the output log of the test run will
+provide the details and summary of the operation.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AbstractWasbTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AbstractWasbTestBase.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AbstractWasbTestBase.java
index d04a19c..0d3a06c 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AbstractWasbTestBase.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AbstractWasbTestBase.java
@@ -18,15 +18,21 @@
package org.apache.hadoop.fs.azure;
-import static org.junit.Assume.assumeNotNull;
+import java.io.IOException;
-import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.conf.Configuration;
import org.junit.After;
import org.junit.Before;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.azure.integration.AzureTestConstants;
+import org.apache.hadoop.io.IOUtils;
+
+import static org.junit.Assume.assumeNotNull;
+import static org.apache.hadoop.fs.azure.integration.AzureTestUtils.*;
+
/**
* Abstract test class that provides basic setup and teardown of testing Azure
* Storage account. Each subclass defines a different set of test cases to run
@@ -34,41 +40,137 @@ import org.slf4j.LoggerFactory;
* to run those tests. The returned account might integrate with Azure Storage
* directly or it might be a mock implementation.
*/
-public abstract class AbstractWasbTestBase {
+public abstract class AbstractWasbTestBase extends AbstractWasbTestWithTimeout
+ implements AzureTestConstants {
protected static final Logger LOG =
LoggerFactory.getLogger(AbstractWasbTestBase.class);
- @VisibleForTesting
protected NativeAzureFileSystem fs;
- private AzureBlobStorageTestAccount testAccount;
+ protected AzureBlobStorageTestAccount testAccount;
@Before
public void setUp() throws Exception {
- testAccount = createTestAccount();
- if (testAccount != null) {
- fs = testAccount.getFileSystem();
- }
- assumeNotNull(testAccount);
+ AzureBlobStorageTestAccount account = createTestAccount();
+ assumeNotNull(account);
+ bindToTestAccount(account);
}
@After
public void tearDown() throws Exception {
- if (testAccount != null) {
- testAccount.cleanup();
- testAccount = null;
- fs = null;
- }
+ describe("closing test account and filesystem");
+ testAccount = cleanupTestAccount(testAccount);
+ IOUtils.closeStream(fs);
+ fs = null;
}
- public Configuration getConfiguration() {
- return new Configuration();
+ /**
+ * Create the configuration to use when creating a test account.
+ * Subclasses can override this to tune the test account configuration.
+ * @return a configuration.
+ */
+ public Configuration createConfiguration() {
+ return AzureBlobStorageTestAccount.createTestConfiguration();
}
+ /**
+ * Create the test account.
+ * Subclasses must implement this.
+ * @return the test account.
+ * @throws Exception
+ */
protected abstract AzureBlobStorageTestAccount createTestAccount()
throws Exception;
+ /**
+ * Get the test account.
+ * @return the current test account.
+ */
protected AzureBlobStorageTestAccount getTestAccount() {
return testAccount;
}
+
+ /**
+ * Get the filesystem
+ * @return the current filesystem.
+ */
+ protected NativeAzureFileSystem getFileSystem() {
+ return fs;
+ }
+
+ /**
+ * Get the configuration used to create the filesystem
+ * @return the configuration of the test FS
+ */
+ protected Configuration getConfiguration() {
+ return getFileSystem().getConf();
+ }
+
+ /**
+ * Bind to a new test account; closing any existing one.
+ * This updates the test account returned in {@link #getTestAccount()}
+ * and the filesystem in {@link #getFileSystem()}.
+ * @param account new test account
+ */
+ protected void bindToTestAccount(AzureBlobStorageTestAccount account) {
+ // clean any existing test account
+ cleanupTestAccount(testAccount);
+ IOUtils.closeStream(fs);
+ testAccount = account;
+ if (testAccount != null) {
+ fs = testAccount.getFileSystem();
+ }
+ }
+
+ /**
+ * Return a path to a blob which will be unique for this fork.
+ * @param filepath filepath
+ * @return a path under the default blob directory
+ * @throws IOException
+ */
+ protected Path blobPath(String filepath) throws IOException {
+ return blobPathForTests(getFileSystem(), filepath);
+ }
+
+ /**
+ * Create a path under the test path provided by
+ * the FS contract.
+ * @param filepath path string in
+ * @return a path qualified by the test filesystem
+ * @throws IOException IO problems
+ */
+ protected Path path(String filepath) throws IOException {
+ return pathForTests(getFileSystem(), filepath);
+ }
+
+ /**
+ * Return a path bonded to this method name, unique to this fork during
+ * parallel execution.
+ * @return a method name unique to (fork, method).
+ * @throws IOException IO problems
+ */
+ protected Path methodPath() throws IOException {
+ return path(methodName.getMethodName());
+ }
+
+ /**
+ * Return a blob path bonded to this method name, unique to this fork during
+ * parallel execution.
+ * @return a method name unique to (fork, method).
+ * @throws IOException IO problems
+ */
+ protected Path methodBlobPath() throws IOException {
+ return blobPath(methodName.getMethodName());
+ }
+
+ /**
+ * Describe a test in the logs.
+ * @param text text to print
+ * @param args arguments to format in the printing
+ */
+ protected void describe(String text, Object... args) {
+ LOG.info("\n\n{}: {}\n",
+ methodName.getMethodName(),
+ String.format(text, args));
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AbstractWasbTestWithTimeout.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AbstractWasbTestWithTimeout.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AbstractWasbTestWithTimeout.java
new file mode 100644
index 0000000..b7076a4
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AbstractWasbTestWithTimeout.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azure;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.rules.TestName;
+import org.junit.rules.Timeout;
+
+import org.apache.hadoop.fs.azure.integration.AzureTestConstants;
+
+/**
+ * Base class for any Wasb test with timeouts & named threads.
+ * This class does not attempt to bind to Azure.
+ */
+public class AbstractWasbTestWithTimeout extends Assert {
+
+ /**
+ * The name of the current method.
+ */
+ @Rule
+ public TestName methodName = new TestName();
+ /**
+ * Set the timeout for every test.
+ * This is driven by the value returned by {@link #getTestTimeoutMillis()}.
+ */
+ @Rule
+ public Timeout testTimeout = new Timeout(getTestTimeoutMillis());
+
+ /**
+ * Name the junit thread for the class. This will overridden
+ * before the individual test methods are run.
+ */
+ @BeforeClass
+ public static void nameTestThread() {
+ Thread.currentThread().setName("JUnit");
+ }
+
+ /**
+ * Name the thread to the current test method.
+ */
+ @Before
+ public void nameThread() {
+ Thread.currentThread().setName("JUnit-" + methodName.getMethodName());
+ }
+
+ /**
+ * Override point: the test timeout in milliseconds.
+ * @return a timeout in milliseconds
+ */
+ protected int getTestTimeoutMillis() {
+ return AzureTestConstants.AZURE_TEST_TIMEOUT;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java
index 7fa59ce..5b36c87 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java
@@ -21,12 +21,15 @@ package org.apache.hadoop.fs.azure;
import com.microsoft.azure.storage.*;
import com.microsoft.azure.storage.blob.*;
import com.microsoft.azure.storage.core.Base64;
-import org.apache.commons.configuration2.SubsetConfiguration;
+import org.junit.Assert;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.commons.configuration2.SubsetConfiguration;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.azure.integration.AzureTestConstants;
import org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation;
import org.apache.hadoop.fs.azure.metrics.AzureFileSystemMetricsSystem;
import org.apache.hadoop.metrics2.AbstractMetric;
@@ -35,6 +38,8 @@ import org.apache.hadoop.metrics2.MetricsSink;
import org.apache.hadoop.metrics2.MetricsTag;
import org.apache.hadoop.metrics2.impl.TestMetricsConfig;
+import java.io.File;
+import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.*;
@@ -46,10 +51,10 @@ import static org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.KEY_USE_SECU
/**
* Helper class to create WASB file systems backed by either a mock in-memory
- * implementation or a real Azure Storage account. See RunningLiveWasbTests.txt
- * for instructions on how to connect to a real Azure Storage account.
+ * implementation or a real Azure Storage account.
*/
-public final class AzureBlobStorageTestAccount {
+public final class AzureBlobStorageTestAccount implements AutoCloseable,
+ AzureTestConstants {
private static final Logger LOG = LoggerFactory.getLogger(
AzureBlobStorageTestAccount.class);
@@ -166,6 +171,7 @@ public final class AzureBlobStorageTestAccount {
return new Path("/" + DEFAULT_PAGE_BLOB_DIRECTORY);
}
+ @Deprecated
public static Path pageBlobPath(String fileName) {
return new Path(pageBlobPath(), fileName);
}
@@ -201,6 +207,9 @@ public final class AzureBlobStorageTestAccount {
* @return
*/
private boolean wasGeneratedByMe(MetricsRecord currentRecord) {
+ Assert.assertNotNull("null filesystem", fs);
+ Assert.assertNotNull("null filesystemn instance ID",
+ fs.getInstrumentation().getFileSystemInstanceId());
String myFsId = fs.getInstrumentation().getFileSystemInstanceId().toString();
for (MetricsTag currentTag : currentRecord.tags()) {
if (currentTag.name().equalsIgnoreCase("wasbFileSystemId")) {
@@ -247,13 +256,16 @@ public final class AzureBlobStorageTestAccount {
getBlobReference(blobKey).releaseLease(accessCondition);
}
- private static void saveMetricsConfigFile() {
+ private static void saveMetricsConfigFile() throws IOException {
if (!metricsConfigSaved) {
+ String testFilename = TestMetricsConfig.getTestFilename(
+ "hadoop-metrics2-azure-file-system");
+ File dest = new File(testFilename).getCanonicalFile();
+ dest.getParentFile().mkdirs();
new org.apache.hadoop.metrics2.impl.ConfigBuilder()
.add("azure-file-system.sink.azuretestcollector.class",
StandardCollector.class.getName())
- .save(TestMetricsConfig.getTestFilename(
- "hadoop-metrics2-azure-file-system.properties"));
+ .save(testFilename);
metricsConfigSaved = true;
}
}
@@ -314,9 +326,8 @@ public final class AzureBlobStorageTestAccount {
Configuration conf = createTestConfiguration();
if (!conf.getBoolean(USE_EMULATOR_PROPERTY_NAME, false)) {
// Not configured to test against the storage emulator.
- LOG.warn("Skipping emulator Azure test because configuration doesn't "
- + "indicate that it's running. Please see RunningLiveWasbTests.txt "
- + "for guidance.");
+ LOG.warn("Skipping emulator Azure test because configuration "
+ + "doesn't indicate that it's running.");
return null;
}
CloudStorageAccount account =
@@ -482,8 +493,7 @@ public final class AzureBlobStorageTestAccount {
credentials = StorageCredentialsAnonymous.ANONYMOUS;
} else {
LOG.warn("Skipping live Azure test because of missing key for"
- + " account '" + accountName + "'. "
- + "Please see RunningLiveWasbTests.txt for guidance.");
+ + " account '" + accountName + "'.");
return null;
}
} else {
@@ -517,8 +527,7 @@ public final class AzureBlobStorageTestAccount {
throws URISyntaxException, KeyProviderException {
String testAccountName = conf.get(TEST_ACCOUNT_NAME_PROPERTY_NAME);
if (testAccountName == null) {
- LOG.warn("Skipping live Azure test because of missing test account. "
- + "Please see RunningLiveWasbTests.txt for guidance.");
+ LOG.warn("Skipping live Azure test because of missing test account");
return null;
}
return createStorageAccount(testAccountName, conf, false);
@@ -863,6 +872,11 @@ public final class AzureBlobStorageTestAccount {
}
}
+ @Override
+ public void close() throws Exception {
+ cleanup();
+ }
+
public NativeAzureFileSystem getFileSystem() {
return fs;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestAzureConcurrentOutOfBandIo.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestAzureConcurrentOutOfBandIo.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestAzureConcurrentOutOfBandIo.java
new file mode 100644
index 0000000..7e733dc
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestAzureConcurrentOutOfBandIo.java
@@ -0,0 +1,179 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azure;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Arrays;
+
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.fs.azure.integration.AzureTestUtils;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.fs.permission.PermissionStatus;
+
+/**
+ * Handle OOB IO into a shared container.
+ */
+public class ITestAzureConcurrentOutOfBandIo extends AbstractWasbTestBase {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ITestAzureConcurrentOutOfBandIo.class);
+
+ // Class constants.
+ static final int DOWNLOAD_BLOCK_SIZE = 8 * 1024 * 1024;
+ static final int UPLOAD_BLOCK_SIZE = 4 * 1024 * 1024;
+ static final int BLOB_SIZE = 32 * 1024 * 1024;
+
+ // Number of blocks to be written before flush.
+ static final int NUMBER_OF_BLOCKS = 2;
+
+ @Override
+ protected AzureBlobStorageTestAccount createTestAccount() throws Exception {
+ return AzureBlobStorageTestAccount.createOutOfBandStore(
+ UPLOAD_BLOCK_SIZE, DOWNLOAD_BLOCK_SIZE);
+ }
+
+ class DataBlockWriter implements Runnable {
+
+ Thread runner;
+ AzureBlobStorageTestAccount writerStorageAccount;
+ String key;
+ boolean done = false;
+
+ /**
+ * Constructor captures the test account.
+ *
+ * @param testAccount
+ */
+ public DataBlockWriter(AzureBlobStorageTestAccount testAccount, String key) {
+ writerStorageAccount = testAccount;
+ this.key = key;
+ }
+
+ /**
+ * Start writing blocks to Azure storage.
+ */
+ public void startWriting() {
+ runner = new Thread(this); // Create the block writer thread.
+ runner.start(); // Start the block writer thread.
+ }
+
+ /**
+ * Stop writing blocks to Azure storage.
+ */
+ public void stopWriting() {
+ done = true;
+ }
+
+ /**
+ * Implementation of the runnable interface. The run method is a tight loop
+ * which repeatedly updates the blob with a 4 MB block.
+ */
+ public void run() {
+ byte[] dataBlockWrite = new byte[UPLOAD_BLOCK_SIZE];
+
+ OutputStream outputStream = null;
+
+ try {
+ for (int i = 0; !done; i++) {
+ // Write two 4 MB blocks to the blob.
+ //
+ outputStream = writerStorageAccount.getStore().storefile(
+ key,
+ new PermissionStatus("", "", FsPermission.getDefault()),
+ key);
+
+ Arrays.fill(dataBlockWrite, (byte) (i % 256));
+ for (int j = 0; j < NUMBER_OF_BLOCKS; j++) {
+ outputStream.write(dataBlockWrite);
+ }
+
+ outputStream.flush();
+ outputStream.close();
+ }
+ } catch (AzureException e) {
+ LOG.error("DatablockWriter thread encountered a storage exception."
+ + e.getMessage(), e);
+ } catch (IOException e) {
+ LOG.error("DatablockWriter thread encountered an I/O exception."
+ + e.getMessage(), e);
+ }
+ }
+ }
+
+ @Test
+ public void testReadOOBWrites() throws Exception {
+
+ byte[] dataBlockWrite = new byte[UPLOAD_BLOCK_SIZE];
+ byte[] dataBlockRead = new byte[UPLOAD_BLOCK_SIZE];
+
+ // Write to blob to make sure it exists.
+ //
+ // Write five 4 MB blocks to the blob. To ensure there is data in the blob before
+ // reading. This eliminates the race between the reader and writer threads.
+ String key = "WASB_String" + AzureTestUtils.getForkID() + ".txt";
+ OutputStream outputStream = testAccount.getStore().storefile(
+ key,
+ new PermissionStatus("", "", FsPermission.getDefault()),
+ key);
+ Arrays.fill(dataBlockWrite, (byte) 255);
+ for (int i = 0; i < NUMBER_OF_BLOCKS; i++) {
+ outputStream.write(dataBlockWrite);
+ }
+
+ outputStream.flush();
+ outputStream.close();
+
+ // Start writing blocks to Azure store using the DataBlockWriter thread.
+ DataBlockWriter writeBlockTask = new DataBlockWriter(testAccount, key);
+ writeBlockTask.startWriting();
+ int count = 0;
+
+ for (int i = 0; i < 5; i++) {
+ try(InputStream inputStream = testAccount.getStore().retrieve(key)) {
+ count = 0;
+ int c = 0;
+
+ while (c >= 0) {
+ c = inputStream.read(dataBlockRead, 0, UPLOAD_BLOCK_SIZE);
+ if (c < 0) {
+ break;
+ }
+
+ // Counting the number of bytes.
+ count += c;
+ }
+ } catch (IOException e) {
+ System.out.println(e.getCause().toString());
+ e.printStackTrace();
+ fail();
+ }
+ }
+
+ // Stop writing blocks.
+ writeBlockTask.stopWriting();
+
+ // Validate that a block was read.
+ assertEquals(NUMBER_OF_BLOCKS * UPLOAD_BLOCK_SIZE, count);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestAzureConcurrentOutOfBandIoWithSecureMode.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestAzureConcurrentOutOfBandIoWithSecureMode.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestAzureConcurrentOutOfBandIoWithSecureMode.java
new file mode 100644
index 0000000..2b0ea56
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestAzureConcurrentOutOfBandIoWithSecureMode.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azure;
+
+/**
+ * Extends ITestAzureConcurrentOutOfBandIo in order to run testReadOOBWrites with secure mode
+ * (fs.azure.secure.mode) both enabled and disabled.
+ */
+public class ITestAzureConcurrentOutOfBandIoWithSecureMode
+ extends ITestAzureConcurrentOutOfBandIo {
+
+ @Override
+ protected AzureBlobStorageTestAccount createTestAccount() throws Exception {
+ return AzureBlobStorageTestAccount.createOutOfBandStore(
+ UPLOAD_BLOCK_SIZE, DOWNLOAD_BLOCK_SIZE, true);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestAzureFileSystemErrorConditions.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestAzureFileSystemErrorConditions.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestAzureFileSystemErrorConditions.java
new file mode 100644
index 0000000..49e6730
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestAzureFileSystemErrorConditions.java
@@ -0,0 +1,243 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azure;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.HttpURLConnection;
+import java.net.URI;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.concurrent.Callable;
+
+import com.microsoft.azure.storage.OperationContext;
+import com.microsoft.azure.storage.SendingRequestEvent;
+import com.microsoft.azure.storage.StorageEvent;
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.TestHookOperationContext;
+import org.apache.hadoop.test.GenericTestUtils;
+
+import static org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.NO_ACCESS_TO_CONTAINER_MSG;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+import static org.junit.Assume.assumeNotNull;
+
+/**
+ * Error handling.
+ */
+public class ITestAzureFileSystemErrorConditions extends
+ AbstractWasbTestWithTimeout {
+ private static final int ALL_THREE_FILE_SIZE = 1024;
+
+ @Test
+ public void testNoInitialize() throws Exception {
+ intercept(AssertionError.class,
+ new Callable<FileMetadata>() {
+ @Override
+ public FileMetadata call() throws Exception {
+ return new AzureNativeFileSystemStore()
+ .retrieveMetadata("foo");
+ }
+ });
+ }
+
+ /**
+ * Try accessing an unauthorized or non-existent (treated the same) container
+ * from WASB.
+ */
+ @Test
+ public void testAccessUnauthorizedPublicContainer() throws Exception {
+ final String container = "nonExistentContainer";
+ final String account = "hopefullyNonExistentAccount";
+ Path noAccessPath = new Path(
+ "wasb://" + container + "@" + account + "/someFile");
+ NativeAzureFileSystem.suppressRetryPolicy();
+ try {
+ FileSystem.get(noAccessPath.toUri(), new Configuration())
+ .open(noAccessPath);
+ assertTrue("Should've thrown.", false);
+ } catch (AzureException ex) {
+ GenericTestUtils.assertExceptionContains(
+ String.format(NO_ACCESS_TO_CONTAINER_MSG, account, container), ex);
+ } finally {
+ NativeAzureFileSystem.resumeRetryPolicy();
+ }
+ }
+
+ @Test
+ public void testAccessContainerWithWrongVersion() throws Exception {
+ AzureNativeFileSystemStore store = new AzureNativeFileSystemStore();
+ MockStorageInterface mockStorage = new MockStorageInterface();
+ store.setAzureStorageInteractionLayer(mockStorage);
+ try (FileSystem fs = new NativeAzureFileSystem(store)) {
+ Configuration conf = new Configuration();
+ AzureBlobStorageTestAccount.setMockAccountKey(conf);
+ HashMap<String, String> metadata = new HashMap<String, String>();
+ metadata.put(AzureNativeFileSystemStore.VERSION_METADATA_KEY,
+ "2090-04-05"); // It's from the future!
+ mockStorage.addPreExistingContainer(
+ AzureBlobStorageTestAccount.getMockContainerUri(), metadata);
+
+ AzureException ex = intercept(AzureException.class,
+ new Callable<FileStatus[]>() {
+ @Override
+ public FileStatus[] call() throws Exception {
+ fs.initialize(new URI(AzureBlobStorageTestAccount.MOCK_WASB_URI),
+ conf);
+ return fs.listStatus(new Path("/"));
+ }
+ });
+ GenericTestUtils.assertExceptionContains(
+ "unsupported version: 2090-04-05.", ex);
+ }
+ }
+
+ private interface ConnectionRecognizer {
+ boolean isTargetConnection(HttpURLConnection connection);
+ }
+
+ private class TransientErrorInjector extends StorageEvent<SendingRequestEvent> {
+ private final ConnectionRecognizer connectionRecognizer;
+ private boolean injectedErrorOnce = false;
+
+ public TransientErrorInjector(ConnectionRecognizer connectionRecognizer) {
+ this.connectionRecognizer = connectionRecognizer;
+ }
+
+ @Override
+ public void eventOccurred(SendingRequestEvent eventArg) {
+ HttpURLConnection connection
+ = (HttpURLConnection) eventArg.getConnectionObject();
+ if (!connectionRecognizer.isTargetConnection(connection)) {
+ return;
+ }
+ if (!injectedErrorOnce) {
+ connection.setReadTimeout(1);
+ connection.disconnect();
+ injectedErrorOnce = true;
+ }
+ }
+ }
+
+ private void injectTransientError(NativeAzureFileSystem fs,
+ final ConnectionRecognizer connectionRecognizer) {
+ fs.getStore().addTestHookToOperationContext(new TestHookOperationContext() {
+ @Override
+ public OperationContext modifyOperationContext(OperationContext original) {
+ original.getSendingRequestEventHandler().addListener(
+ new TransientErrorInjector(connectionRecognizer));
+ return original;
+ }
+ });
+ }
+
+ @Test
+ public void testTransientErrorOnDelete() throws Exception {
+ // Need to do this test against a live storage account
+ AzureBlobStorageTestAccount testAccount =
+ AzureBlobStorageTestAccount.create();
+ assumeNotNull(testAccount);
+ try {
+ NativeAzureFileSystem fs = testAccount.getFileSystem();
+ injectTransientError(fs, new ConnectionRecognizer() {
+ @Override
+ public boolean isTargetConnection(HttpURLConnection connection) {
+ return connection.getRequestMethod().equals("DELETE");
+ }
+ });
+ Path testFile = new Path("/a/b");
+ assertTrue(fs.createNewFile(testFile));
+ assertTrue(fs.rename(testFile, new Path("/x")));
+ } finally {
+ testAccount.cleanup();
+ }
+ }
+
+ private void writeAllThreeFile(NativeAzureFileSystem fs, Path testFile)
+ throws IOException {
+ byte[] buffer = new byte[ALL_THREE_FILE_SIZE];
+ Arrays.fill(buffer, (byte) 3);
+ try(OutputStream stream = fs.create(testFile)) {
+ stream.write(buffer);
+ }
+ }
+
+ private void readAllThreeFile(NativeAzureFileSystem fs, Path testFile)
+ throws IOException {
+ byte[] buffer = new byte[ALL_THREE_FILE_SIZE];
+ InputStream inStream = fs.open(testFile);
+ assertEquals(buffer.length,
+ inStream.read(buffer, 0, buffer.length));
+ inStream.close();
+ for (int i = 0; i < buffer.length; i++) {
+ assertEquals(3, buffer[i]);
+ }
+ }
+
+ @Test
+ public void testTransientErrorOnCommitBlockList() throws Exception {
+ // Need to do this test against a live storage account
+ AzureBlobStorageTestAccount testAccount =
+ AzureBlobStorageTestAccount.create();
+ assumeNotNull(testAccount);
+ try {
+ NativeAzureFileSystem fs = testAccount.getFileSystem();
+ injectTransientError(fs, new ConnectionRecognizer() {
+ @Override
+ public boolean isTargetConnection(HttpURLConnection connection) {
+ return connection.getRequestMethod().equals("PUT")
+ && connection.getURL().getQuery() != null
+ && connection.getURL().getQuery().contains("blocklist");
+ }
+ });
+ Path testFile = new Path("/a/b");
+ writeAllThreeFile(fs, testFile);
+ readAllThreeFile(fs, testFile);
+ } finally {
+ testAccount.cleanup();
+ }
+ }
+
+ @Test
+ public void testTransientErrorOnRead() throws Exception {
+ // Need to do this test against a live storage account
+ AzureBlobStorageTestAccount testAccount =
+ AzureBlobStorageTestAccount.create();
+ assumeNotNull(testAccount);
+ try {
+ NativeAzureFileSystem fs = testAccount.getFileSystem();
+ Path testFile = new Path("/a/b");
+ writeAllThreeFile(fs, testFile);
+ injectTransientError(fs, new ConnectionRecognizer() {
+ @Override
+ public boolean isTargetConnection(HttpURLConnection connection) {
+ return connection.getRequestMethod().equals("GET");
+ }
+ });
+ readAllThreeFile(fs, testFile);
+ } finally {
+ testAccount.cleanup();
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org