You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2017/09/15 16:39:30 UTC
[05/20] hadoop git commit: HADOOP-14553. Add (parallelized)
integration tests to hadoop-azure Contributed by Steve Loughran
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestContainerChecks.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestContainerChecks.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestContainerChecks.java
deleted file mode 100644
index f6ab94d..0000000
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestContainerChecks.java
+++ /dev/null
@@ -1,185 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assume.assumeNotNull;
-
-import java.io.FileNotFoundException;
-import java.util.EnumSet;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount.CreateOptions;
-import org.junit.After;
-import org.junit.Assume;
-import org.junit.Before;
-import org.junit.Test;
-
-import com.microsoft.azure.storage.blob.BlobOutputStream;
-import com.microsoft.azure.storage.blob.CloudBlobContainer;
-import com.microsoft.azure.storage.blob.CloudBlockBlob;
-
-/**
- * Tests that WASB creates containers only if needed.
- */
-public class TestContainerChecks {
- private AzureBlobStorageTestAccount testAccount;
- private boolean runningInSASMode = false;
- @After
- public void tearDown() throws Exception {
- if (testAccount != null) {
- testAccount.cleanup();
- testAccount = null;
- }
- }
-
- @Before
- public void setMode() {
- runningInSASMode = AzureBlobStorageTestAccount.createTestConfiguration().
- getBoolean(AzureNativeFileSystemStore.KEY_USE_SECURE_MODE, false);
- }
-
- @Test
- public void testContainerExistAfterDoesNotExist() throws Exception {
- testAccount = AzureBlobStorageTestAccount.create("",
- EnumSet.noneOf(CreateOptions.class));
- assumeNotNull(testAccount);
- CloudBlobContainer container = testAccount.getRealContainer();
- FileSystem fs = testAccount.getFileSystem();
-
- // Starting off with the container not there
- assertFalse(container.exists());
-
- // A list shouldn't create the container and will set file system store
- // state to DoesNotExist
- try {
- fs.listStatus(new Path("/"));
- assertTrue("Should've thrown.", false);
- } catch (FileNotFoundException ex) {
- assertTrue("Unexpected exception: " + ex,
- ex.getMessage().contains("does not exist."));
- }
- assertFalse(container.exists());
-
- // Create a container outside of the WASB FileSystem
- container.create();
- // Add a file to the container outside of the WASB FileSystem
- CloudBlockBlob blob = testAccount.getBlobReference("foo");
- BlobOutputStream outputStream = blob.openOutputStream();
- outputStream.write(new byte[10]);
- outputStream.close();
-
- // Make sure the file is visible
- assertTrue(fs.exists(new Path("/foo")));
- assertTrue(container.exists());
- }
-
- @Test
- public void testContainerCreateAfterDoesNotExist() throws Exception {
- testAccount = AzureBlobStorageTestAccount.create("",
- EnumSet.noneOf(CreateOptions.class));
- assumeNotNull(testAccount);
- CloudBlobContainer container = testAccount.getRealContainer();
- FileSystem fs = testAccount.getFileSystem();
-
- // Starting off with the container not there
- assertFalse(container.exists());
-
- // A list shouldn't create the container and will set file system store
- // state to DoesNotExist
- try {
- assertNull(fs.listStatus(new Path("/")));
- assertTrue("Should've thrown.", false);
- } catch (FileNotFoundException ex) {
- assertTrue("Unexpected exception: " + ex,
- ex.getMessage().contains("does not exist."));
- }
- assertFalse(container.exists());
-
- // Create a container outside of the WASB FileSystem
- container.create();
-
- // Write should succeed
- assertTrue(fs.createNewFile(new Path("/foo")));
- assertTrue(container.exists());
- }
-
- @Test
- public void testContainerCreateOnWrite() throws Exception {
- testAccount = AzureBlobStorageTestAccount.create("",
- EnumSet.noneOf(CreateOptions.class));
- assumeNotNull(testAccount);
- CloudBlobContainer container = testAccount.getRealContainer();
- FileSystem fs = testAccount.getFileSystem();
-
- // Starting off with the container not there
- assertFalse(container.exists());
-
- // A list shouldn't create the container.
- try {
- fs.listStatus(new Path("/"));
- assertTrue("Should've thrown.", false);
- } catch (FileNotFoundException ex) {
- assertTrue("Unexpected exception: " + ex,
- ex.getMessage().contains("does not exist."));
- }
- assertFalse(container.exists());
-
- // Neither should a read.
- try {
- fs.open(new Path("/foo"));
- assertFalse("Should've thrown.", true);
- } catch (FileNotFoundException ex) {
- }
- assertFalse(container.exists());
-
- // Neither should a rename
- assertFalse(fs.rename(new Path("/foo"), new Path("/bar")));
- assertFalse(container.exists());
-
- // But a write should.
- assertTrue(fs.createNewFile(new Path("/foo")));
- assertTrue(container.exists());
- }
-
- @Test
- public void testContainerChecksWithSas() throws Exception {
-
- Assume.assumeFalse(runningInSASMode);
- testAccount = AzureBlobStorageTestAccount.create("",
- EnumSet.of(CreateOptions.UseSas));
- assumeNotNull(testAccount);
- CloudBlobContainer container = testAccount.getRealContainer();
- FileSystem fs = testAccount.getFileSystem();
-
- // The container shouldn't be there
- assertFalse(container.exists());
-
- // A write should just fail
- try {
- fs.createNewFile(new Path("/foo"));
- assertFalse("Should've thrown.", true);
- } catch (AzureException ex) {
- }
- assertFalse(container.exists());
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationExceptionHandling.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationExceptionHandling.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationExceptionHandling.java
deleted file mode 100644
index 9ac25dd..0000000
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationExceptionHandling.java
+++ /dev/null
@@ -1,269 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import java.io.FileNotFoundException;
-
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsAction;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Test;
-
-
-public class TestFileSystemOperationExceptionHandling
- extends AbstractWasbTestBase {
-
- private FSDataInputStream inputStream = null;
-
- private static Path testPath = new Path("testfile.dat");
-
- private static Path testFolderPath = new Path("testfolder");
-
- /*
- * Helper method that creates a InputStream to validate exceptions
- * for various scenarios
- */
- private void setupInputStreamToTest(AzureBlobStorageTestAccount testAccount)
- throws Exception {
-
- FileSystem fs = testAccount.getFileSystem();
-
- // Step 1: Create a file and write dummy data.
- Path testFilePath1 = new Path("test1.dat");
- Path testFilePath2 = new Path("test2.dat");
- FSDataOutputStream outputStream = fs.create(testFilePath1);
- String testString = "This is a test string";
- outputStream.write(testString.getBytes());
- outputStream.close();
-
- // Step 2: Open a read stream on the file.
- inputStream = fs.open(testFilePath1);
-
- // Step 3: Rename the file
- fs.rename(testFilePath1, testFilePath2);
- }
-
- /*
- * Tests a basic single threaded read scenario for Page blobs.
- */
- @Test(expected=FileNotFoundException.class)
- public void testSingleThreadedPageBlobReadScenario() throws Throwable {
- AzureBlobStorageTestAccount testAccount = ExceptionHandlingTestHelper.getPageBlobTestStorageAccount();
- setupInputStreamToTest(testAccount);
- byte[] readBuffer = new byte[512];
- inputStream.read(readBuffer);
- }
-
- /*
- * Tests a basic single threaded seek scenario for Page blobs.
- */
- @Test(expected=FileNotFoundException.class)
- public void testSingleThreadedPageBlobSeekScenario() throws Throwable {
- AzureBlobStorageTestAccount testAccount = ExceptionHandlingTestHelper.getPageBlobTestStorageAccount();
- setupInputStreamToTest(testAccount);
- inputStream.seek(5);
- }
-
- /*
- * Test a basic single thread seek scenario for Block blobs.
- */
- @Test(expected=FileNotFoundException.class)
- public void testSingleThreadBlockBlobSeekScenario() throws Throwable {
-
- AzureBlobStorageTestAccount testAccount = createTestAccount();
- setupInputStreamToTest(testAccount);
- inputStream.seek(5);
- inputStream.read();
- }
-
- /*
- * Tests a basic single threaded read scenario for Block blobs.
- */
- @Test(expected=FileNotFoundException.class)
- public void testSingledThreadBlockBlobReadScenario() throws Throwable{
- AzureBlobStorageTestAccount testAccount = createTestAccount();
- setupInputStreamToTest(testAccount);
- byte[] readBuffer = new byte[512];
- inputStream.read(readBuffer);
- }
-
- @Test(expected=FileNotFoundException.class)
- /*
- * Tests basic single threaded setPermission scenario
- */
- public void testSingleThreadedBlockBlobSetPermissionScenario() throws Throwable {
-
- ExceptionHandlingTestHelper.createEmptyFile(createTestAccount(), testPath);
- fs.delete(testPath, true);
- fs.setPermission(testPath, new FsPermission(FsAction.EXECUTE, FsAction.READ, FsAction.READ));
- }
-
- @Test(expected=FileNotFoundException.class)
- /*
- * Tests basic single threaded setPermission scenario
- */
- public void testSingleThreadedPageBlobSetPermissionScenario() throws Throwable {
- ExceptionHandlingTestHelper.createEmptyFile(ExceptionHandlingTestHelper.getPageBlobTestStorageAccount(),
- testPath);
- fs.delete(testPath, true);
- fs.setOwner(testPath, "testowner", "testgroup");
- }
-
- @Test(expected=FileNotFoundException.class)
- /*
- * Tests basic single threaded setPermission scenario
- */
- public void testSingleThreadedBlockBlobSetOwnerScenario() throws Throwable {
-
- ExceptionHandlingTestHelper.createEmptyFile(createTestAccount(), testPath);
- fs.delete(testPath, true);
- fs.setOwner(testPath, "testowner", "testgroup");
- }
-
- @Test(expected=FileNotFoundException.class)
- /*
- * Tests basic single threaded setPermission scenario
- */
- public void testSingleThreadedPageBlobSetOwnerScenario() throws Throwable {
- ExceptionHandlingTestHelper.createEmptyFile(ExceptionHandlingTestHelper.getPageBlobTestStorageAccount(),
- testPath);
- fs.delete(testPath, true);
- fs.setPermission(testPath, new FsPermission(FsAction.EXECUTE, FsAction.READ, FsAction.READ));
- }
-
- @Test(expected=FileNotFoundException.class)
- /*
- * Test basic single threaded listStatus scenario
- */
- public void testSingleThreadedBlockBlobListStatusScenario() throws Throwable {
- ExceptionHandlingTestHelper.createTestFolder(createTestAccount(), testFolderPath);
- fs.delete(testFolderPath, true);
- fs.listStatus(testFolderPath);
- }
-
- @Test(expected=FileNotFoundException.class)
- /*
- * Test basica single threaded listStatus scenario
- */
- public void testSingleThreadedPageBlobListStatusScenario() throws Throwable {
- ExceptionHandlingTestHelper.createTestFolder(ExceptionHandlingTestHelper.getPageBlobTestStorageAccount(),
- testFolderPath);
- fs.delete(testFolderPath, true);
- fs.listStatus(testFolderPath);
- }
-
- @Test
- /*
- * Test basic single threaded listStatus scenario
- */
- public void testSingleThreadedBlockBlobRenameScenario() throws Throwable {
-
- ExceptionHandlingTestHelper.createEmptyFile(createTestAccount(),
- testPath);
- Path dstPath = new Path("dstFile.dat");
- fs.delete(testPath, true);
- boolean renameResult = fs.rename(testPath, dstPath);
- Assert.assertFalse(renameResult);
- }
-
- @Test
- /*
- * Test basic single threaded listStatus scenario
- */
- public void testSingleThreadedPageBlobRenameScenario() throws Throwable {
-
- ExceptionHandlingTestHelper.createEmptyFile(ExceptionHandlingTestHelper.getPageBlobTestStorageAccount(),
- testPath);
- Path dstPath = new Path("dstFile.dat");
- fs.delete(testPath, true);
- boolean renameResult = fs.rename(testPath, dstPath);
- Assert.assertFalse(renameResult);
- }
-
- @Test
- /*
- * Test basic single threaded listStatus scenario
- */
- public void testSingleThreadedBlockBlobDeleteScenario() throws Throwable {
-
- ExceptionHandlingTestHelper.createEmptyFile(createTestAccount(),
- testPath);
- fs.delete(testPath, true);
- boolean deleteResult = fs.delete(testPath, true);
- Assert.assertFalse(deleteResult);
- }
-
- @Test
- /*
- * Test basic single threaded listStatus scenario
- */
- public void testSingleThreadedPageBlobDeleteScenario() throws Throwable {
-
- ExceptionHandlingTestHelper.createEmptyFile(ExceptionHandlingTestHelper.getPageBlobTestStorageAccount(),
- testPath);
- fs.delete(testPath, true);
- boolean deleteResult = fs.delete(testPath, true);
- Assert.assertFalse(deleteResult);
- }
-
- @Test(expected=FileNotFoundException.class)
- /*
- * Test basic single threaded listStatus scenario
- */
- public void testSingleThreadedBlockBlobOpenScenario() throws Throwable {
-
- ExceptionHandlingTestHelper.createEmptyFile(createTestAccount(),
- testPath);
- fs.delete(testPath, true);
- inputStream = fs.open(testPath);
- }
-
- @Test(expected=FileNotFoundException.class)
- /*
- * Test basic single threaded listStatus scenario
- */
- public void testSingleThreadedPageBlobOpenScenario() throws Throwable {
-
- ExceptionHandlingTestHelper.createEmptyFile(ExceptionHandlingTestHelper.getPageBlobTestStorageAccount(),
- testPath);
- fs.delete(testPath, true);
- inputStream = fs.open(testPath);
- }
-
- @After
- public void tearDown() throws Exception {
- if (inputStream != null) {
- inputStream.close();
- }
-
- if (fs != null && fs.exists(testPath)) {
- fs.delete(testPath, true);
- }
- }
-
- @Override
- protected AzureBlobStorageTestAccount createTestAccount() throws Exception {
- return AzureBlobStorageTestAccount.create();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationExceptionMessage.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationExceptionMessage.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationExceptionMessage.java
deleted file mode 100644
index e619817..0000000
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationExceptionMessage.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-import java.net.URI;
-import java.util.UUID;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.test.GenericTestUtils;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-import static org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.NO_ACCESS_TO_CONTAINER_MSG;
-
-
-public class TestFileSystemOperationExceptionMessage extends
- NativeAzureFileSystemBaseTest {
-
- @Test
- public void testAnonymouseCredentialExceptionMessage() throws Throwable{
-
- Configuration conf = AzureBlobStorageTestAccount.createTestConfiguration();
- String testStorageAccount = conf.get("fs.azure.test.account.name");
- conf = new Configuration();
- conf.set("fs.AbstractFileSystem.wasb.impl", "org.apache.hadoop.fs.azure.Wasb");
- conf.set("fs.azure.skip.metrics", "true");
-
- String testContainer = UUID.randomUUID().toString();
- String wasbUri = String.format("wasb://%s@%s",
- testContainer, testStorageAccount);
-
- fs = new NativeAzureFileSystem();
- try {
- fs.initialize(new URI(wasbUri), conf);
- } catch (Exception ex) {
-
- Throwable innerException = ex.getCause();
- while (innerException != null
- && !(innerException instanceof AzureException)) {
- innerException = innerException.getCause();
- }
-
- if (innerException != null) {
- String exceptionMessage = innerException.getMessage();
- if (exceptionMessage == null
- || exceptionMessage.length() == 0) {
- Assert.fail();}
- else {
- GenericTestUtils.assertExceptionContains(String.format(
- NO_ACCESS_TO_CONTAINER_MSG, testStorageAccount, testContainer),
- ex);
- }
- } else {
- Assert.fail();
- }
- }
- }
-
- @Override
- protected AzureBlobStorageTestAccount createTestAccount() throws Exception {
- return AzureBlobStorageTestAccount.create();
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationsExceptionHandlingMultiThreaded.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationsExceptionHandlingMultiThreaded.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationsExceptionHandlingMultiThreaded.java
deleted file mode 100644
index 1cd18ee..0000000
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationsExceptionHandlingMultiThreaded.java
+++ /dev/null
@@ -1,330 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import java.io.FileNotFoundException;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsAction;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.junit.After;
-import org.junit.Test;
-
-public class TestFileSystemOperationsExceptionHandlingMultiThreaded
- extends AbstractWasbTestBase {
-
- FSDataInputStream inputStream = null;
-
- private static Path testPath = new Path("testfile.dat");
- private static Path testFolderPath = new Path("testfolder");
-
-
- /*
- * Helper method to creates an input stream to test various scenarios.
- */
- private void getInputStreamToTest(FileSystem fs, Path testPath) throws Throwable {
-
- FSDataOutputStream outputStream = fs.create(testPath);
- String testString = "This is a test string";
- outputStream.write(testString.getBytes());
- outputStream.close();
-
- inputStream = fs.open(testPath);
- }
-
- /*
- * Test to validate correct exception is thrown for Multithreaded read
- * scenario for block blobs
- */
- @Test(expected=FileNotFoundException.class)
- public void testMultiThreadedBlockBlobReadScenario() throws Throwable {
-
- AzureBlobStorageTestAccount testAccount = createTestAccount();
- fs = testAccount.getFileSystem();
- Path testFilePath1 = new Path("test1.dat");
-
- getInputStreamToTest(fs, testFilePath1);
- Thread renameThread = new Thread(new RenameThread(fs, testFilePath1));
- renameThread.start();
-
- renameThread.join();
-
- byte[] readBuffer = new byte[512];
- inputStream.read(readBuffer);
- }
-
- /*
- * Test to validate correct exception is thrown for Multithreaded seek
- * scenario for block blobs
- */
-
- @Test(expected=FileNotFoundException.class)
- public void testMultiThreadBlockBlobSeekScenario() throws Throwable {
-
- AzureBlobStorageTestAccount testAccount = createTestAccount();
- fs = testAccount.getFileSystem();
- Path testFilePath1 = new Path("test1.dat");
-
- getInputStreamToTest(fs, testFilePath1);
- Thread renameThread = new Thread(new RenameThread(fs, testFilePath1));
- renameThread.start();
-
- renameThread.join();
-
- inputStream.seek(5);
- inputStream.read();
- }
-
- @Test(expected=FileNotFoundException.class)
- /*
- * Tests basic multi threaded setPermission scenario
- */
- public void testMultiThreadedPageBlobSetPermissionScenario() throws Throwable {
- ExceptionHandlingTestHelper.createEmptyFile(ExceptionHandlingTestHelper.getPageBlobTestStorageAccount(),
- testPath);
- Thread t = new Thread(new DeleteThread(fs, testPath));
- t.start();
- while (t.isAlive()) {
- fs.setPermission(testPath, new FsPermission(FsAction.EXECUTE, FsAction.READ, FsAction.READ));
- }
- fs.setPermission(testPath, new FsPermission(FsAction.EXECUTE, FsAction.READ, FsAction.READ));
- }
-
- @Test(expected=FileNotFoundException.class)
- /*
- * Tests basic multi threaded setPermission scenario
- */
- public void testMultiThreadedBlockBlobSetPermissionScenario() throws Throwable {
- ExceptionHandlingTestHelper.createEmptyFile(createTestAccount(),
- testPath);
- Thread t = new Thread(new DeleteThread(fs, testPath));
- t.start();
- while (t.isAlive()) {
- fs.setPermission(testPath, new FsPermission(FsAction.EXECUTE, FsAction.READ, FsAction.READ));
- }
- fs.setPermission(testPath, new FsPermission(FsAction.EXECUTE, FsAction.READ, FsAction.READ));
- }
-
- @Test(expected=FileNotFoundException.class)
- /*
- * Tests basic multi threaded setPermission scenario
- */
- public void testMultiThreadedPageBlobOpenScenario() throws Throwable {
-
- ExceptionHandlingTestHelper.createEmptyFile(createTestAccount(),
- testPath);
- Thread t = new Thread(new DeleteThread(fs, testPath));
- t.start();
- while (t.isAlive()) {
- inputStream = fs.open(testPath);
- inputStream.close();
- }
-
- inputStream = fs.open(testPath);
- inputStream.close();
- }
-
- @Test(expected=FileNotFoundException.class)
- /*
- * Tests basic multi threaded setPermission scenario
- */
- public void testMultiThreadedBlockBlobOpenScenario() throws Throwable {
-
- ExceptionHandlingTestHelper.createEmptyFile(ExceptionHandlingTestHelper.getPageBlobTestStorageAccount(),
- testPath);
- Thread t = new Thread(new DeleteThread(fs, testPath));
- t.start();
-
- while (t.isAlive()) {
- inputStream = fs.open(testPath);
- inputStream.close();
- }
- inputStream = fs.open(testPath);
- inputStream.close();
- }
-
- @Test(expected=FileNotFoundException.class)
- /*
- * Tests basic multi threaded setOwner scenario
- */
- public void testMultiThreadedBlockBlobSetOwnerScenario() throws Throwable {
-
- ExceptionHandlingTestHelper.createEmptyFile(createTestAccount(), testPath);
- Thread t = new Thread(new DeleteThread(fs, testPath));
- t.start();
- while (t.isAlive()) {
- fs.setOwner(testPath, "testowner", "testgroup");
- }
- fs.setOwner(testPath, "testowner", "testgroup");
- }
-
- @Test(expected=FileNotFoundException.class)
- /*
- * Tests basic multi threaded setOwner scenario
- */
- public void testMultiThreadedPageBlobSetOwnerScenario() throws Throwable {
- ExceptionHandlingTestHelper.createEmptyFile(ExceptionHandlingTestHelper.getPageBlobTestStorageAccount(),
- testPath);
- Thread t = new Thread(new DeleteThread(fs, testPath));
- t.start();
- while (t.isAlive()) {
- fs.setOwner(testPath, "testowner", "testgroup");
- }
- fs.setOwner(testPath, "testowner", "testgroup");
- }
-
- @Test(expected=FileNotFoundException.class)
- /*
- * Tests basic multi threaded listStatus scenario
- */
- public void testMultiThreadedBlockBlobListStatusScenario() throws Throwable {
-
- ExceptionHandlingTestHelper.createTestFolder(createTestAccount(), testFolderPath);
- Thread t = new Thread(new DeleteThread(fs, testFolderPath));
- t.start();
- while (t.isAlive()) {
- fs.listStatus(testFolderPath);
- }
- fs.listStatus(testFolderPath);
- }
-
- @Test(expected=FileNotFoundException.class)
- /*
- * Tests basic multi threaded listStatus scenario
- */
- public void testMultiThreadedPageBlobListStatusScenario() throws Throwable {
-
- ExceptionHandlingTestHelper.createTestFolder(ExceptionHandlingTestHelper.getPageBlobTestStorageAccount(),
- testFolderPath);
- Thread t = new Thread(new DeleteThread(fs, testFolderPath));
- t.start();
- while (t.isAlive()) {
- fs.listStatus(testFolderPath);
- }
- fs.listStatus(testFolderPath);
- }
-
- /*
- * Test to validate correct exception is thrown for Multithreaded read
- * scenario for page blobs
- */
-
- @Test(expected=FileNotFoundException.class)
- public void testMultiThreadedPageBlobReadScenario() throws Throwable {
-
- AzureBlobStorageTestAccount testAccount = ExceptionHandlingTestHelper.getPageBlobTestStorageAccount();
- fs = testAccount.getFileSystem();
- Path testFilePath1 = new Path("test1.dat");
-
- getInputStreamToTest(fs, testFilePath1);
- Thread renameThread = new Thread(new RenameThread(fs, testFilePath1));
- renameThread.start();
-
- renameThread.join();
- byte[] readBuffer = new byte[512];
- inputStream.read(readBuffer);
- }
-
- /*
- * Test to validate correct exception is thrown for Multithreaded seek
- * scenario for page blobs
- */
-
- @Test(expected=FileNotFoundException.class)
- public void testMultiThreadedPageBlobSeekScenario() throws Throwable {
-
- AzureBlobStorageTestAccount testAccount = ExceptionHandlingTestHelper.getPageBlobTestStorageAccount();
- fs = testAccount.getFileSystem();
- Path testFilePath1 = new Path("test1.dat");
-
- getInputStreamToTest(fs, testFilePath1);
- Thread renameThread = new Thread(new RenameThread(fs, testFilePath1));
- renameThread.start();
-
- renameThread.join();
- inputStream.seek(5);
- }
-
- @Override
- protected AzureBlobStorageTestAccount createTestAccount() throws Exception {
- return AzureBlobStorageTestAccount.create();
- }
-
- @After
- public void tearDown() throws Exception {
-
- if (inputStream != null) {
- inputStream.close();
- }
-
- if (fs != null && fs.exists(testPath)) {
- fs.delete(testPath, true);
- }
- }
-}
-
-/*
- * Helper thread that just renames the test file.
- */
-class RenameThread implements Runnable {
-
- private FileSystem fs;
- private Path testPath;
- private Path renamePath = new Path("test2.dat");
-
- public RenameThread(FileSystem fs, Path testPath) {
- this.fs = fs;
- this.testPath = testPath;
- }
-
- @Override
- public void run(){
- try {
- fs.rename(testPath, renamePath);
- }catch (Exception e) {
- // Swallowing the exception as the
- // correctness of the test is controlled
- // by the other thread
- }
- }
-}
-
-class DeleteThread implements Runnable {
- private FileSystem fs;
- private Path testPath;
-
- public DeleteThread(FileSystem fs, Path testPath) {
- this.fs = fs;
- this.testPath = testPath;
- }
-
- @Override
- public void run() {
- try {
- fs.delete(testPath, true);
- } catch (Exception e) {
- // Swallowing the exception as the
- // correctness of the test is controlled
- // by the other thread
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationsWithThreads.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationsWithThreads.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationsWithThreads.java
deleted file mode 100644
index fd3690c..0000000
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationsWithThreads.java
+++ /dev/null
@@ -1,821 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import java.io.IOException;
-import java.net.URI;
-import java.util.concurrent.RejectedExecutionException;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.commons.logging.impl.Log4JLogger;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.azure.NativeAzureFileSystem.FolderRenamePending;
-import org.apache.hadoop.test.GenericTestUtils.LogCapturer;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-import org.mockito.Mockito;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
-
-/**
- * Tests the Native Azure file system (WASB) using parallel threads for rename and delete operations.
- */
-public class TestFileSystemOperationsWithThreads extends AbstractWasbTestBase {
-
- private final int renameThreads = 10;
- private final int deleteThreads = 20;
- private int iterations = 1;
- private LogCapturer logs = null;
-
- @Rule
- public ExpectedException exception = ExpectedException.none();
-
- @Before
- public void setUp() throws Exception {
- super.setUp();
- Configuration conf = fs.getConf();
-
- // By default enable parallel threads for rename and delete operations.
- // Also enable flat listing of blobs for these operations.
- conf.setInt(NativeAzureFileSystem.AZURE_RENAME_THREADS, renameThreads);
- conf.setInt(NativeAzureFileSystem.AZURE_DELETE_THREADS, deleteThreads);
- conf.setBoolean(AzureNativeFileSystemStore.KEY_ENABLE_FLAT_LISTING, true);
-
- URI uri = fs.getUri();
- fs.initialize(uri, conf);
-
- // Capture logs
- logs = LogCapturer.captureLogs(new Log4JLogger(org.apache.log4j.Logger
- .getRootLogger()));
- }
-
- /*
- * Helper method to create sub directory and different types of files
- * for multiple iterations.
- */
- private void createFolder(FileSystem fs, String root) throws Exception {
- fs.mkdirs(new Path(root));
- for (int i = 0; i < this.iterations; i++) {
- fs.mkdirs(new Path(root + "/" + i));
- fs.createNewFile(new Path(root + "/" + i + "/fileToRename"));
- fs.createNewFile(new Path(root + "/" + i + "/file/to/rename"));
- fs.createNewFile(new Path(root + "/" + i + "/file+to%rename"));
- fs.createNewFile(new Path(root + "/fileToRename" + i));
- }
- }
-
- /*
- * Helper method to do rename operation and validate all files in source folder
- * doesn't exists and similar files exists in new folder.
- */
- private void validateRenameFolder(FileSystem fs, String source, String dest) throws Exception {
- // Create source folder with files.
- createFolder(fs, source);
- Path sourceFolder = new Path(source);
- Path destFolder = new Path(dest);
-
- // rename operation
- assertTrue(fs.rename(sourceFolder, destFolder));
- assertTrue(fs.exists(destFolder));
-
- for (int i = 0; i < this.iterations; i++) {
- // Check destination folder and files exists.
- assertTrue(fs.exists(new Path(dest + "/" + i)));
- assertTrue(fs.exists(new Path(dest + "/" + i + "/fileToRename")));
- assertTrue(fs.exists(new Path(dest + "/" + i + "/file/to/rename")));
- assertTrue(fs.exists(new Path(dest + "/" + i + "/file+to%rename")));
- assertTrue(fs.exists(new Path(dest + "/fileToRename" + i)));
-
- // Check source folder and files doesn't exists.
- assertFalse(fs.exists(new Path(source + "/" + i)));
- assertFalse(fs.exists(new Path(source + "/" + i + "/fileToRename")));
- assertFalse(fs.exists(new Path(source + "/" + i + "/file/to/rename")));
- assertFalse(fs.exists(new Path(source + "/" + i + "/file+to%rename")));
- assertFalse(fs.exists(new Path(source + "/fileToRename" + i)));
- }
- }
-
- /*
- * Test case for rename operation with multiple threads and flat listing enabled.
- */
- @Test
- public void testRenameSmallFolderWithThreads() throws Exception {
-
- validateRenameFolder(fs, "root", "rootnew");
-
- // With single iteration, we would have created 7 blobs.
- int expectedThreadsCreated = Math.min(7, renameThreads);
-
- // Validate from logs that threads are created.
- String content = logs.getOutput();
- assertInLog(content, "ms with threads: " + expectedThreadsCreated);
-
- // Validate thread executions
- for (int i = 0; i < expectedThreadsCreated; i++) {
- assertInLog(content,
- "AzureBlobRenameThread-" + Thread.currentThread().getName() + "-" + i);
- }
-
- // Also ensure that we haven't spawned extra threads.
- if (expectedThreadsCreated < renameThreads) {
- for (int i = expectedThreadsCreated; i < renameThreads; i++) {
- assertNotInLog(content,
- "AzureBlobRenameThread-" + Thread.currentThread().getName() + "-" + i);
- }
- }
- }
-
- /*
- * Test case for rename operation with multiple threads and flat listing enabled.
- */
- @Test
- public void testRenameLargeFolderWithThreads() throws Exception {
-
- // Populate source folder with large number of files and directories.
- this.iterations = 10;
- validateRenameFolder(fs, "root", "rootnew");
-
- // Validate from logs that threads are created.
- String content = logs.getOutput();
- assertInLog(content, "ms with threads: " + renameThreads);
-
- // Validate thread executions
- for (int i = 0; i < renameThreads; i++) {
- assertInLog(content,
- "AzureBlobRenameThread-" + Thread.currentThread().getName() + "-" + i);
- }
- }
-
- /*
- * Test case for rename operation with threads disabled and flat listing enabled.
- */
- @Test
- public void testRenameLargeFolderDisableThreads() throws Exception {
- Configuration conf = fs.getConf();
-
- // Number of threads set to 0 or 1 disables threads.
- conf.setInt(NativeAzureFileSystem.AZURE_RENAME_THREADS, 0);
- URI uri = fs.getUri();
- fs.initialize(uri, conf);
-
- // Populate source folder with large number of files and directories.
- this.iterations = 10;
- validateRenameFolder(fs, "root", "rootnew");
-
- // Validate from logs that threads are disabled.
- String content = logs.getOutput();
- assertInLog(content,
- "Disabling threads for Rename operation as thread count 0");
-
- // Validate no thread executions
- for (int i = 0; i < renameThreads; i++) {
- String term = "AzureBlobRenameThread-"
- + Thread.currentThread().getName()
- + "-" + i;
- assertNotInLog(content, term);
- }
- }
-
- /**
- * Assert that a log contains the given term.
- * @param content log output
- * @param term search term
- */
- protected void assertInLog(String content, String term) {
- assertTrue("Empty log", !content.isEmpty());
- if (!content.contains(term)) {
- String message = "No " + term + " found in logs";
- LOG.error(message);
- System.err.println(content);
- fail(message);
- }
- }
-
- /**
- * Assert that a log does not contain the given term.
- * @param content log output
- * @param term search term
- */
- protected void assertNotInLog(String content, String term) {
- assertTrue("Empty log", !content.isEmpty());
- if (content.contains(term)) {
- String message = term + " found in logs";
- LOG.error(message);
- System.err.println(content);
- fail(message);
- }
- }
-
- /*
- * Test case for rename operation with threads and flat listing disabled.
- */
- @Test
- public void testRenameSmallFolderDisableThreadsDisableFlatListing() throws Exception {
- Configuration conf = fs.getConf();
- conf = fs.getConf();
-
- // Number of threads set to 0 or 1 disables threads.
- conf.setInt(NativeAzureFileSystem.AZURE_RENAME_THREADS, 1);
- conf.setBoolean(AzureNativeFileSystemStore.KEY_ENABLE_FLAT_LISTING, false);
- URI uri = fs.getUri();
- fs.initialize(uri, conf);
-
- validateRenameFolder(fs, "root", "rootnew");
-
- // Validate from logs that threads are disabled.
- String content = logs.getOutput();
- assertInLog(content,
- "Disabling threads for Rename operation as thread count 1");
-
- // Validate no thread executions
- for (int i = 0; i < renameThreads; i++) {
- assertNotInLog(content,
- "AzureBlobRenameThread-" + Thread.currentThread().getName() + "-" + i);
- }
- }
-
- /*
- * Helper method to do delete operation and validate all files in source folder
- * doesn't exists after delete operation.
- */
- private void validateDeleteFolder(FileSystem fs, String source) throws Exception {
- // Create folder with files.
- createFolder(fs, "root");
- Path sourceFolder = new Path(source);
-
- // Delete operation
- assertTrue(fs.delete(sourceFolder, true));
- assertFalse(fs.exists(sourceFolder));
-
- for (int i = 0; i < this.iterations; i++) {
- // check that source folder and files doesn't exists
- assertFalse(fs.exists(new Path(source + "/" + i)));
- assertFalse(fs.exists(new Path(source + "/" + i + "/fileToRename")));
- assertFalse(fs.exists(new Path(source + "/" + i + "/file/to/rename")));
- assertFalse(fs.exists(new Path(source + "/" + i + "/file+to%rename")));
- assertFalse(fs.exists(new Path(source + "/fileToRename" + i)));
- }
- }
-
- /*
- * Test case for delete operation with multiple threads and flat listing enabled.
- */
- @Test
- public void testDeleteSmallFolderWithThreads() throws Exception {
-
- validateDeleteFolder(fs, "root");
-
- // With single iteration, we would have created 7 blobs.
- int expectedThreadsCreated = Math.min(7, deleteThreads);
-
- // Validate from logs that threads are enabled.
- String content = logs.getOutput();
- assertInLog(content, "ms with threads: " + expectedThreadsCreated);
-
- // Validate thread executions
- for (int i = 0; i < expectedThreadsCreated; i++) {
- assertInLog(content,
- "AzureBlobDeleteThread-" + Thread.currentThread().getName() + "-" + i);
- }
-
- // Also ensure that we haven't spawned extra threads.
- if (expectedThreadsCreated < deleteThreads) {
- for (int i = expectedThreadsCreated; i < deleteThreads; i++) {
- assertNotInLog(content,
- "AzureBlobDeleteThread-" + Thread.currentThread().getName() + "-" + i);
- }
- }
- }
-
- /*
- * Test case for delete operation with multiple threads and flat listing enabled.
- */
- @Test
- public void testDeleteLargeFolderWithThreads() throws Exception {
- // Populate source folder with large number of files and directories.
- this.iterations = 10;
- validateDeleteFolder(fs, "root");
-
- // Validate from logs that threads are enabled.
- String content = logs.getOutput();
- assertInLog(content, "ms with threads: " + deleteThreads);
-
- // Validate thread executions
- for (int i = 0; i < deleteThreads; i++) {
- assertInLog(content,
- "AzureBlobDeleteThread-" + Thread.currentThread().getName() + "-" + i);
- }
- }
-
- /*
- * Test case for delete operation with threads disabled and flat listing enabled.
- */
- @Test
- public void testDeleteLargeFolderDisableThreads() throws Exception {
- Configuration conf = fs.getConf();
- conf.setInt(NativeAzureFileSystem.AZURE_DELETE_THREADS, 0);
- URI uri = fs.getUri();
- fs.initialize(uri, conf);
-
- // Populate source folder with large number of files and directories.
- this.iterations = 10;
- validateDeleteFolder(fs, "root");
-
- // Validate from logs that threads are disabled.
- String content = logs.getOutput();
- assertInLog(content,
- "Disabling threads for Delete operation as thread count 0");
-
- // Validate no thread executions
- for (int i = 0; i < deleteThreads; i++) {
- assertNotInLog(content,
- "AzureBlobDeleteThread-" + Thread.currentThread().getName() + "-" + i);
- }
- }
-
- /*
- * Test case for rename operation with threads and flat listing disabled.
- */
- @Test
- public void testDeleteSmallFolderDisableThreadsDisableFlatListing() throws Exception {
- Configuration conf = fs.getConf();
-
- // Number of threads set to 0 or 1 disables threads.
- conf.setInt(NativeAzureFileSystem.AZURE_DELETE_THREADS, 1);
- conf.setBoolean(AzureNativeFileSystemStore.KEY_ENABLE_FLAT_LISTING, false);
- URI uri = fs.getUri();
- fs.initialize(uri, conf);
-
- validateDeleteFolder(fs, "root");
-
- // Validate from logs that threads are disabled.
- String content = logs.getOutput();
- assertInLog(content,
- "Disabling threads for Delete operation as thread count 1");
-
- // Validate no thread executions
- for (int i = 0; i < deleteThreads; i++) {
- assertNotInLog(content,
- "AzureBlobDeleteThread-" + Thread.currentThread().getName() + "-" + i);
- }
- }
-
- /*
- * Test case for delete operation with multiple threads and flat listing enabled.
- */
- @Test
- public void testDeleteThreadPoolExceptionFailure() throws Exception {
-
- // Spy azure file system object and raise exception for new thread pool
- NativeAzureFileSystem mockFs = Mockito.spy((NativeAzureFileSystem) fs);
-
- String path = mockFs.pathToKey(mockFs.makeAbsolute(new Path("root")));
-
- AzureFileSystemThreadPoolExecutor mockThreadPoolExecutor = Mockito.spy(
- mockFs.getThreadPoolExecutor(deleteThreads, "AzureBlobDeleteThread", "Delete",
- path, NativeAzureFileSystem.AZURE_DELETE_THREADS));
- Mockito.when(mockThreadPoolExecutor.getThreadPool(7)).thenThrow(new Exception());
-
- // With single iteration, we would have created 7 blobs resulting 7 threads.
- Mockito.when(mockFs.getThreadPoolExecutor(deleteThreads, "AzureBlobDeleteThread", "Delete",
- path, NativeAzureFileSystem.AZURE_DELETE_THREADS)).thenReturn(mockThreadPoolExecutor);
-
- validateDeleteFolder(mockFs, "root");
-
- // Validate from logs that threads are disabled.
- String content = logs.getOutput();
- assertInLog(content, "Failed to create thread pool with threads");
- assertInLog(content, "Serializing the Delete operation");
- }
-
- /*
- * Test case for delete operation with multiple threads and flat listing enabled.
- */
- @Test
- public void testDeleteThreadPoolExecuteFailure() throws Exception {
-
- // Mock thread pool executor to throw exception for all requests.
- ThreadPoolExecutor mockThreadExecutor = Mockito.mock(ThreadPoolExecutor.class);
- Mockito.doThrow(new RejectedExecutionException()).when(mockThreadExecutor).execute(Mockito.any(Runnable.class));
-
- // Spy azure file system object and return mocked thread pool
- NativeAzureFileSystem mockFs = Mockito.spy((NativeAzureFileSystem) fs);
-
- String path = mockFs.pathToKey(mockFs.makeAbsolute(new Path("root")));
-
- AzureFileSystemThreadPoolExecutor mockThreadPoolExecutor = Mockito.spy(
- mockFs.getThreadPoolExecutor(deleteThreads, "AzureBlobDeleteThread", "Delete",
- path, NativeAzureFileSystem.AZURE_DELETE_THREADS));
- Mockito.when(mockThreadPoolExecutor.getThreadPool(7)).thenReturn(mockThreadExecutor);
-
- // With single iteration, we would have created 7 blobs resulting 7 threads.
- Mockito.when(mockFs.getThreadPoolExecutor(deleteThreads, "AzureBlobDeleteThread", "Delete",
- path, NativeAzureFileSystem.AZURE_DELETE_THREADS)).thenReturn(mockThreadPoolExecutor);
-
- validateDeleteFolder(mockFs, "root");
-
- // Validate from logs that threads are disabled.
- String content = logs.getOutput();
- assertInLog(content,
- "Rejected execution of thread for Delete operation on blob");
- assertInLog(content, "Serializing the Delete operation");
- }
-
- /*
- * Test case for delete operation with multiple threads and flat listing enabled.
- */
- @Test
- public void testDeleteThreadPoolExecuteSingleThreadFailure() throws Exception {
-
- // Spy azure file system object and return mocked thread pool
- NativeAzureFileSystem mockFs = Mockito.spy((NativeAzureFileSystem) fs);
-
- // Spy a thread pool executor and link it to azure file system object.
- String path = mockFs.pathToKey(mockFs.makeAbsolute(new Path("root")));
- AzureFileSystemThreadPoolExecutor mockThreadPoolExecutor = Mockito.spy(
- mockFs.getThreadPoolExecutor(deleteThreads, "AzureBlobDeleteThread", "Delete",
- path, NativeAzureFileSystem.AZURE_DELETE_THREADS));
-
- // With single iteration, we would have created 7 blobs resulting 7 threads.
- Mockito.when(mockFs.getThreadPoolExecutor(deleteThreads, "AzureBlobDeleteThread", "Delete",
- path, NativeAzureFileSystem.AZURE_DELETE_THREADS)).thenReturn(mockThreadPoolExecutor);
-
- // Create a thread executor and link it to mocked thread pool executor object.
- ThreadPoolExecutor mockThreadExecutor = Mockito.spy(mockThreadPoolExecutor.getThreadPool(7));
- Mockito.when(mockThreadPoolExecutor.getThreadPool(7)).thenReturn(mockThreadExecutor);
-
- // Mock thread executor to throw exception for all requests.
- Mockito.doCallRealMethod().doThrow(new RejectedExecutionException()).when(mockThreadExecutor).execute(Mockito.any(Runnable.class));
-
- validateDeleteFolder(mockFs, "root");
-
- // Validate from logs that threads are enabled and unused threads.
- String content = logs.getOutput();
- assertInLog(content,
- "Using thread pool for Delete operation with threads 7");
- assertInLog(content,
- "6 threads not used for Delete operation on blob");
- }
-
- /*
- * Test case for delete operation with multiple threads and flat listing enabled.
- */
- @Test
- public void testDeleteThreadPoolTerminationFailure() throws Exception {
-
- // Spy azure file system object and return mocked thread pool
- NativeAzureFileSystem mockFs = Mockito.spy((NativeAzureFileSystem) fs);
-
- // Spy a thread pool executor and link it to azure file system object.
- String path = mockFs.pathToKey(mockFs.makeAbsolute(new Path("root")));
- AzureFileSystemThreadPoolExecutor mockThreadPoolExecutor = Mockito.spy(
- ((NativeAzureFileSystem) fs).getThreadPoolExecutor(deleteThreads, "AzureBlobDeleteThread", "Delete",
- path, NativeAzureFileSystem.AZURE_DELETE_THREADS));
-
- // Create a thread executor and link it to mocked thread pool executor object.
- // Mock thread executor to throw exception for terminating threads.
- ThreadPoolExecutor mockThreadExecutor = Mockito.mock(ThreadPoolExecutor.class);
- Mockito.doNothing().when(mockThreadExecutor).execute(Mockito.any(Runnable.class));
- Mockito.when(mockThreadExecutor.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS)).thenThrow(new InterruptedException());
-
- Mockito.when(mockThreadPoolExecutor.getThreadPool(7)).thenReturn(mockThreadExecutor);
-
- // With single iteration, we would have created 7 blobs resulting 7 threads.
- Mockito.when(mockFs.getThreadPoolExecutor(deleteThreads, "AzureBlobDeleteThread", "Delete",
- path, NativeAzureFileSystem.AZURE_DELETE_THREADS)).thenReturn(mockThreadPoolExecutor);
-
- createFolder(mockFs, "root");
- Path sourceFolder = new Path("root");
- boolean exception = false;
- try {
- mockFs.delete(sourceFolder, true);
- } catch (IOException e){
- exception = true;
- }
-
- assertTrue(exception);
- assertTrue(mockFs.exists(sourceFolder));
-
- // Validate from logs that threads are enabled and delete operation is failed.
- String content = logs.getOutput();
- assertInLog(content,
- "Using thread pool for Delete operation with threads");
- assertInLog(content, "Threads got interrupted Delete blob operation");
- assertInLog(content,
- "Delete failed as operation on subfolders and files failed.");
- }
-
- /*
- * Validate that when a directory is deleted recursively, the operation succeeds
- * even if a child directory delete fails because the directory does not exist.
- * This can happen if a child directory is deleted by an external agent while
- * the parent is in progress of being deleted recursively.
- */
- @Test
- public void testRecursiveDirectoryDeleteWhenChildDirectoryDeleted()
- throws Exception {
- testRecusiveDirectoryDelete(true);
- }
-
- /*
- * Validate that when a directory is deleted recursively, the operation succeeds
- * even if a file delete fails because it does not exist.
- * This can happen if a file is deleted by an external agent while
- * the parent directory is in progress of being deleted.
- */
- @Test
- public void testRecursiveDirectoryDeleteWhenDeletingChildFileReturnsFalse()
- throws Exception {
- testRecusiveDirectoryDelete(false);
- }
-
- private void testRecusiveDirectoryDelete(boolean useDir) throws Exception {
- String childPathToBeDeletedByExternalAgent = (useDir)
- ? "root/0"
- : "root/0/fileToRename";
- // Spy azure file system object and return false for deleting one file
- NativeAzureFileSystem mockFs = Mockito.spy((NativeAzureFileSystem) fs);
- String path = mockFs.pathToKey(mockFs.makeAbsolute(new Path(
- childPathToBeDeletedByExternalAgent)));
-
- Answer<Boolean> answer = new Answer<Boolean>() {
- public Boolean answer(InvocationOnMock invocation) throws Throwable {
- String path = (String) invocation.getArguments()[0];
- boolean isDir = (boolean) invocation.getArguments()[1];
- boolean realResult = fs.deleteFile(path, isDir);
- assertTrue(realResult);
- boolean fakeResult = false;
- return fakeResult;
- }
- };
-
- Mockito.when(mockFs.deleteFile(path, useDir)).thenAnswer(answer);
-
- createFolder(mockFs, "root");
- Path sourceFolder = new Path("root");
-
- assertTrue(mockFs.delete(sourceFolder, true));
- assertFalse(mockFs.exists(sourceFolder));
-
- // Validate from logs that threads are enabled, that a child directory was
- // deleted by an external caller, and the parent delete operation still
- // succeeds.
- String content = logs.getOutput();
- assertInLog(content,
- "Using thread pool for Delete operation with threads");
- assertInLog(content, String.format("Attempt to delete non-existent %s %s",
- useDir ? "directory" : "file", path));
- }
-
- /*
- * Test case for delete operation with multiple threads and flat listing enabled.
- */
- @Test
- public void testDeleteSingleDeleteException() throws Exception {
-
- // Spy azure file system object and raise exception for deleting one file
- NativeAzureFileSystem mockFs = Mockito.spy((NativeAzureFileSystem) fs);
- String path = mockFs.pathToKey(mockFs.makeAbsolute(new Path("root/0")));
- Mockito.doThrow(new IOException()).when(mockFs).deleteFile(path, true);
-
- createFolder(mockFs, "root");
- Path sourceFolder = new Path("root");
-
- boolean exception = false;
- try {
- mockFs.delete(sourceFolder, true);
- } catch (IOException e){
- exception = true;
- }
-
- assertTrue(exception);
- assertTrue(mockFs.exists(sourceFolder));
-
- // Validate from logs that threads are enabled and delete operation failed.
- String content = logs.getOutput();
- assertInLog(content,
- "Using thread pool for Delete operation with threads");
- assertInLog(content,
- "Encountered Exception for Delete operation for file " + path);
- assertInLog(content,
- "Terminating execution of Delete operation now as some other thread already got exception or operation failed");
- }
-
- /*
- * Test case for rename operation with multiple threads and flat listing enabled.
- */
- @Test
- public void testRenameThreadPoolExceptionFailure() throws Exception {
-
- // Spy azure file system object and raise exception for new thread pool
- NativeAzureFileSystem mockFs = Mockito.spy((NativeAzureFileSystem) fs);
-
- String path = mockFs.pathToKey(mockFs.makeAbsolute(new Path("root")));
- AzureFileSystemThreadPoolExecutor mockThreadPoolExecutor = Mockito.spy(
- ((NativeAzureFileSystem) fs).getThreadPoolExecutor(renameThreads, "AzureBlobRenameThread", "Rename",
- path, NativeAzureFileSystem.AZURE_RENAME_THREADS));
- Mockito.when(mockThreadPoolExecutor.getThreadPool(7)).thenThrow(new Exception());
-
- // With single iteration, we would have created 7 blobs resulting 7 threads.
- Mockito.doReturn(mockThreadPoolExecutor).when(mockFs).getThreadPoolExecutor(renameThreads, "AzureBlobRenameThread", "Rename",
- path, NativeAzureFileSystem.AZURE_RENAME_THREADS);
-
- validateRenameFolder(mockFs, "root", "rootnew");
-
- // Validate from logs that threads are disabled.
- String content = logs.getOutput();
- assertInLog(content, "Failed to create thread pool with threads");
- assertInLog(content, "Serializing the Rename operation");
- }
-
- /*
- * Test case for rename operation with multiple threads and flat listing enabled.
- */
- @Test
- public void testRenameThreadPoolExecuteFailure() throws Exception {
-
- // Mock thread pool executor to throw exception for all requests.
- ThreadPoolExecutor mockThreadExecutor = Mockito.mock(ThreadPoolExecutor.class);
- Mockito.doThrow(new RejectedExecutionException()).when(mockThreadExecutor).execute(Mockito.any(Runnable.class));
-
- // Spy azure file system object and return mocked thread pool
- NativeAzureFileSystem mockFs = Mockito.spy((NativeAzureFileSystem) fs);
-
- String path = mockFs.pathToKey(mockFs.makeAbsolute(new Path("root")));
- AzureFileSystemThreadPoolExecutor mockThreadPoolExecutor = Mockito.spy(
- mockFs.getThreadPoolExecutor(renameThreads, "AzureBlobRenameThread", "Rename",
- path, NativeAzureFileSystem.AZURE_RENAME_THREADS));
- Mockito.when(mockThreadPoolExecutor.getThreadPool(7)).thenReturn(mockThreadExecutor);
-
- // With single iteration, we would have created 7 blobs resulting 7 threads.
- Mockito.when(mockFs.getThreadPoolExecutor(renameThreads, "AzureBlobRenameThread", "Rename",
- path, NativeAzureFileSystem.AZURE_RENAME_THREADS)).thenReturn(mockThreadPoolExecutor);
-
- validateRenameFolder(mockFs, "root", "rootnew");
-
- // Validate from logs that threads are disabled.
- String content = logs.getOutput();
- assertInLog(content,
- "Rejected execution of thread for Rename operation on blob");
- assertInLog(content, "Serializing the Rename operation");
- }
-
- /*
- * Test case for rename operation with multiple threads and flat listing enabled.
- */
- @Test
- public void testRenameThreadPoolExecuteSingleThreadFailure() throws Exception {
-
- // Spy azure file system object and return mocked thread pool
- NativeAzureFileSystem mockFs = Mockito.spy((NativeAzureFileSystem) fs);
-
- // Spy a thread pool executor and link it to azure file system object.
- String path = mockFs.pathToKey(mockFs.makeAbsolute(new Path("root")));
- AzureFileSystemThreadPoolExecutor mockThreadPoolExecutor = Mockito.spy(
- mockFs.getThreadPoolExecutor(renameThreads, "AzureBlobRenameThread", "Rename",
- path, NativeAzureFileSystem.AZURE_RENAME_THREADS));
-
- // With single iteration, we would have created 7 blobs resulting 7 threads.
- Mockito.when(mockFs.getThreadPoolExecutor(renameThreads, "AzureBlobRenameThread", "Rename",
- path, NativeAzureFileSystem.AZURE_RENAME_THREADS)).thenReturn(mockThreadPoolExecutor);
-
- // Create a thread executor and link it to mocked thread pool executor object.
- ThreadPoolExecutor mockThreadExecutor = Mockito.spy(mockThreadPoolExecutor.getThreadPool(7));
- Mockito.when(mockThreadPoolExecutor.getThreadPool(7)).thenReturn(mockThreadExecutor);
-
- // Mock thread executor to throw exception for all requests.
- Mockito.doCallRealMethod().doThrow(new RejectedExecutionException()).when(mockThreadExecutor).execute(Mockito.any(Runnable.class));
-
- validateRenameFolder(mockFs, "root", "rootnew");
-
- // Validate from logs that threads are enabled and unused threads exists.
- String content = logs.getOutput();
- assertInLog(content,
- "Using thread pool for Rename operation with threads 7");
- assertInLog(content,
- "6 threads not used for Rename operation on blob");
- }
-
- /*
- * Test case for rename operation with multiple threads and flat listing enabled.
- */
- @Test
- public void testRenameThreadPoolTerminationFailure() throws Exception {
-
- // Spy azure file system object and return mocked thread pool
- NativeAzureFileSystem mockFs = Mockito.spy((NativeAzureFileSystem) fs);
-
- // Spy a thread pool executor and link it to azure file system object.
- String path = mockFs.pathToKey(mockFs.makeAbsolute(new Path("root")));
- AzureFileSystemThreadPoolExecutor mockThreadPoolExecutor = Mockito.spy(
- mockFs.getThreadPoolExecutor(renameThreads, "AzureBlobRenameThread", "Rename",
- path, NativeAzureFileSystem.AZURE_RENAME_THREADS));
-
- // With single iteration, we would have created 7 blobs resulting 7 threads.
- Mockito.when(mockFs.getThreadPoolExecutor(renameThreads, "AzureBlobRenameThread", "Rename",
- path, NativeAzureFileSystem.AZURE_RENAME_THREADS)).thenReturn(mockThreadPoolExecutor);
-
- // Mock thread executor to throw exception for all requests.
- ThreadPoolExecutor mockThreadExecutor = Mockito.mock(ThreadPoolExecutor.class);
- Mockito.doNothing().when(mockThreadExecutor).execute(Mockito.any(Runnable.class));
- Mockito.when(mockThreadExecutor.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS)).thenThrow(new InterruptedException());
- Mockito.when(mockThreadPoolExecutor.getThreadPool(7)).thenReturn(mockThreadExecutor);
-
-
- createFolder(mockFs, "root");
- Path sourceFolder = new Path("root");
- Path destFolder = new Path("rootnew");
- boolean exception = false;
- try {
- mockFs.rename(sourceFolder, destFolder);
- } catch (IOException e){
- exception = true;
- }
-
- assertTrue(exception);
- assertTrue(mockFs.exists(sourceFolder));
-
- // Validate from logs that threads are enabled and rename operation is failed.
- String content = logs.getOutput();
- assertInLog(content,
- "Using thread pool for Rename operation with threads");
- assertInLog(content, "Threads got interrupted Rename blob operation");
- assertInLog(content,
- "Rename failed as operation on subfolders and files failed.");
- }
-
- /*
- * Test case for rename operation with multiple threads and flat listing enabled.
- */
- @Test
- public void testRenameSingleRenameException() throws Exception {
-
- // Spy azure file system object and raise exception for deleting one file
- Path sourceFolder = new Path("root");
- Path destFolder = new Path("rootnew");
-
- // Spy azure file system object and populate rename pending spy object.
- NativeAzureFileSystem mockFs = Mockito.spy((NativeAzureFileSystem) fs);
-
- // Populate data now only such that rename pending spy object would see this data.
- createFolder(mockFs, "root");
-
- String srcKey = mockFs.pathToKey(mockFs.makeAbsolute(sourceFolder));
- String dstKey = mockFs.pathToKey(mockFs.makeAbsolute(destFolder));
-
- FolderRenamePending mockRenameFs = Mockito.spy(mockFs.prepareAtomicFolderRename(srcKey, dstKey));
- Mockito.when(mockFs.prepareAtomicFolderRename(srcKey, dstKey)).thenReturn(mockRenameFs);
- String path = mockFs.pathToKey(mockFs.makeAbsolute(new Path("root/0")));
- Mockito.doThrow(new IOException()).when(mockRenameFs).renameFile(Mockito.any(FileMetadata.class));
-
- boolean exception = false;
- try {
- mockFs.rename(sourceFolder, destFolder);
- } catch (IOException e){
- exception = true;
- }
-
- assertTrue(exception);
- assertTrue(mockFs.exists(sourceFolder));
-
- // Validate from logs that threads are enabled and delete operation failed.
- String content = logs.getOutput();
- assertInLog(content,
- "Using thread pool for Rename operation with threads");
- assertInLog(content,
- "Encountered Exception for Rename operation for file " + path);
- assertInLog(content,
- "Terminating execution of Rename operation now as some other thread already got exception or operation failed");
- }
-
- @Override
- protected AzureBlobStorageTestAccount createTestAccount() throws Exception {
- return AzureBlobStorageTestAccount.create();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFSAuthWithBlobSpecificKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFSAuthWithBlobSpecificKeys.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFSAuthWithBlobSpecificKeys.java
deleted file mode 100644
index 6149154..0000000
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFSAuthWithBlobSpecificKeys.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import org.apache.hadoop.conf.Configuration;
-
-import static org.apache.hadoop.fs.azure.SecureStorageInterfaceImpl.KEY_USE_CONTAINER_SASKEY_FOR_ALL_ACCESS;
-
-/**
- * Test class to hold all WASB authorization tests that use blob-specific keys
- * to access storage.
- */
-public class TestNativeAzureFSAuthWithBlobSpecificKeys
- extends TestNativeAzureFileSystemAuthorizationWithOwner {
-
- @Override
- public Configuration getConfiguration() {
- Configuration conf = super.getConfiguration();
- conf.set(KEY_USE_CONTAINER_SASKEY_FOR_ALL_ACCESS, "false");
- return conf;
- }
-
- @Override
- protected AzureBlobStorageTestAccount createTestAccount() throws Exception {
- Configuration conf = getConfiguration();
- return AzureBlobStorageTestAccount.create(conf);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFSAuthorizationCaching.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFSAuthorizationCaching.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFSAuthorizationCaching.java
deleted file mode 100644
index 84558f8..0000000
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFSAuthorizationCaching.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.ContractTestUtils;
-import org.junit.Test;
-
-import static org.apache.hadoop.fs.azure.CachingAuthorizer.KEY_AUTH_SERVICE_CACHING_ENABLE;
-
-/**
- * Test class to hold all WASB authorization caching related tests.
- */
-public class TestNativeAzureFSAuthorizationCaching
- extends TestNativeAzureFileSystemAuthorizationWithOwner {
-
- private static final int DUMMY_TTL_VALUE = 5000;
-
- @Override
- public Configuration getConfiguration() {
- Configuration conf = super.getConfiguration();
- conf.set(KEY_AUTH_SERVICE_CACHING_ENABLE, "true");
- return conf;
- }
-
- @Override
- protected AzureBlobStorageTestAccount createTestAccount() throws Exception {
- Configuration conf = getConfiguration();
- return AzureBlobStorageTestAccount.create(conf);
- }
-
- /**
- * Test to verify cache behavior -- assert that PUT overwrites value if present
- */
- @Test
- public void testCachePut() throws Throwable {
- CachingAuthorizer<String, Integer> cache = new CachingAuthorizer<>(DUMMY_TTL_VALUE, "TEST");
- cache.init(getConfiguration());
- cache.put("TEST", 1);
- cache.put("TEST", 3);
- int result = cache.get("TEST");
- ContractTestUtils.assertTrue("Cache returned unexpected result", result == 3);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFSPageBlobLive.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFSPageBlobLive.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFSPageBlobLive.java
deleted file mode 100644
index 208cff3..0000000
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFSPageBlobLive.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.azure;
-
-import org.apache.hadoop.conf.Configuration;
-
-/**
- * Run the base Azure file system tests strictly on page blobs to make sure fundamental
- * operations on page blob files and folders work as expected.
- * These operations include create, delete, rename, list, and so on.
- */
-public class TestNativeAzureFSPageBlobLive extends
- NativeAzureFileSystemBaseTest {
-
- @Override
- protected AzureBlobStorageTestAccount createTestAccount()
- throws Exception {
- Configuration conf = new Configuration();
-
- // Configure the page blob directories key so every file created is a page blob.
- conf.set(AzureNativeFileSystemStore.KEY_PAGE_BLOB_DIRECTORIES, "/");
-
- // Configure the atomic rename directories key so every folder will have
- // atomic rename applied.
- conf.set(AzureNativeFileSystemStore.KEY_ATOMIC_RENAME_DIRECTORIES, "/");
- return AzureBlobStorageTestAccount.create(conf);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAppend.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAppend.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAppend.java
deleted file mode 100644
index a2b35cb..0000000
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAppend.java
+++ /dev/null
@@ -1,362 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import java.io.IOException;
-import java.net.URI;
-import java.util.Arrays;
-
-import org.apache.commons.lang.RandomStringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-public class TestNativeAzureFileSystemAppend extends AbstractWasbTestBase {
-
- private static final String TEST_FILE = "test.dat";
- private static final Path TEST_PATH = new Path(TEST_FILE);
-
- private AzureBlobStorageTestAccount testAccount = null;
-
- @Before
- public void setUp() throws Exception {
- super.setUp();
- testAccount = createTestAccount();
- fs = testAccount.getFileSystem();
- Configuration conf = fs.getConf();
- conf.setBoolean(NativeAzureFileSystem.APPEND_SUPPORT_ENABLE_PROPERTY_NAME, true);
- URI uri = fs.getUri();
- fs.initialize(uri, conf);
- }
-
- /*
- * Helper method that creates test data of size provided by the
- * "size" parameter.
- */
- private static byte[] getTestData(int size) {
- byte[] testData = new byte[size];
- System.arraycopy(RandomStringUtils.randomAlphabetic(size).getBytes(), 0, testData, 0, size);
- return testData;
- }
-
- // Helper method to create file and write fileSize bytes of data on it.
- private byte[] createBaseFileWithData(int fileSize, Path testPath) throws Throwable {
-
- FSDataOutputStream createStream = null;
- try {
- createStream = fs.create(testPath);
- byte[] fileData = null;
-
- if (fileSize != 0) {
- fileData = getTestData(fileSize);
- createStream.write(fileData);
- }
- return fileData;
- } finally {
- if (createStream != null) {
- createStream.close();
- }
- }
- }
-
- /*
- * Helper method to verify a file data equal to "dataLength" parameter
- */
- private boolean verifyFileData(int dataLength, byte[] testData, int testDataIndex,
- FSDataInputStream srcStream) {
-
- try {
-
- byte[] fileBuffer = new byte[dataLength];
- byte[] testDataBuffer = new byte[dataLength];
-
- int fileBytesRead = srcStream.read(fileBuffer);
-
- if (fileBytesRead < dataLength) {
- return false;
- }
-
- System.arraycopy(testData, testDataIndex, testDataBuffer, 0, dataLength);
-
- if (!Arrays.equals(fileBuffer, testDataBuffer)) {
- return false;
- }
-
- return true;
-
- } catch (Exception ex) {
- return false;
- }
-
- }
-
- /*
- * Helper method to verify Append on a testFile.
- */
- private boolean verifyAppend(byte[] testData, Path testFile) {
-
- FSDataInputStream srcStream = null;
- try {
-
- srcStream = fs.open(testFile);
- int baseBufferSize = 2048;
- int testDataSize = testData.length;
- int testDataIndex = 0;
-
- while (testDataSize > baseBufferSize) {
-
- if (!verifyFileData(baseBufferSize, testData, testDataIndex, srcStream)) {
- return false;
- }
- testDataIndex += baseBufferSize;
- testDataSize -= baseBufferSize;
- }
-
- if (!verifyFileData(testDataSize, testData, testDataIndex, srcStream)) {
- return false;
- }
-
- return true;
- } catch(Exception ex) {
- return false;
- } finally {
- if (srcStream != null) {
- try {
- srcStream.close();
- } catch(IOException ioe) {
- // Swallowing
- }
- }
- }
- }
-
- /*
- * Test case to verify if an append on small size data works. This tests
- * append E2E
- */
- @Test
- public void testSingleAppend() throws Throwable{
-
- FSDataOutputStream appendStream = null;
- try {
- int baseDataSize = 50;
- byte[] baseDataBuffer = createBaseFileWithData(baseDataSize, TEST_PATH);
-
- int appendDataSize = 20;
- byte[] appendDataBuffer = getTestData(appendDataSize);
- appendStream = fs.append(TEST_PATH, 10);
- appendStream.write(appendDataBuffer);
- appendStream.close();
- byte[] testData = new byte[baseDataSize + appendDataSize];
- System.arraycopy(baseDataBuffer, 0, testData, 0, baseDataSize);
- System.arraycopy(appendDataBuffer, 0, testData, baseDataSize, appendDataSize);
-
- Assert.assertTrue(verifyAppend(testData, TEST_PATH));
- } finally {
- if (appendStream != null) {
- appendStream.close();
- }
- }
- }
-
- /*
- * Test case to verify append to an empty file.
- */
- @Test
- public void testSingleAppendOnEmptyFile() throws Throwable {
-
- FSDataOutputStream appendStream = null;
-
- try {
- createBaseFileWithData(0, TEST_PATH);
-
- int appendDataSize = 20;
- byte[] appendDataBuffer = getTestData(appendDataSize);
- appendStream = fs.append(TEST_PATH, 10);
- appendStream.write(appendDataBuffer);
- appendStream.close();
-
- Assert.assertTrue(verifyAppend(appendDataBuffer, TEST_PATH));
- } finally {
- if (appendStream != null) {
- appendStream.close();
- }
- }
- }
-
- /*
- * Test to verify that we can open only one Append stream on a File.
- */
- @Test
- public void testSingleAppenderScenario() throws Throwable {
-
- FSDataOutputStream appendStream1 = null;
- FSDataOutputStream appendStream2 = null;
- IOException ioe = null;
- try {
- createBaseFileWithData(0, TEST_PATH);
- appendStream1 = fs.append(TEST_PATH, 10);
- boolean encounteredException = false;
- try {
- appendStream2 = fs.append(TEST_PATH, 10);
- } catch(IOException ex) {
- encounteredException = true;
- ioe = ex;
- }
-
- appendStream1.close();
-
- Assert.assertTrue(encounteredException);
- GenericTestUtils.assertExceptionContains("Unable to set Append lease on the Blob", ioe);
- } finally {
- if (appendStream1 != null) {
- appendStream1.close();
- }
-
- if (appendStream2 != null) {
- appendStream2.close();
- }
- }
- }
-
- /*
- * Tests to verify multiple appends on a Blob.
- */
- @Test
- public void testMultipleAppends() throws Throwable {
-
- int baseDataSize = 50;
- byte[] baseDataBuffer = createBaseFileWithData(baseDataSize, TEST_PATH);
-
- int appendDataSize = 100;
- int targetAppendCount = 50;
- byte[] testData = new byte[baseDataSize + (appendDataSize*targetAppendCount)];
- int testDataIndex = 0;
- System.arraycopy(baseDataBuffer, 0, testData, testDataIndex, baseDataSize);
- testDataIndex += baseDataSize;
-
- int appendCount = 0;
-
- FSDataOutputStream appendStream = null;
-
- try {
- while (appendCount < targetAppendCount) {
-
- byte[] appendDataBuffer = getTestData(appendDataSize);
- appendStream = fs.append(TEST_PATH, 30);
- appendStream.write(appendDataBuffer);
- appendStream.close();
-
- System.arraycopy(appendDataBuffer, 0, testData, testDataIndex, appendDataSize);
- testDataIndex += appendDataSize;
- appendCount++;
- }
-
- Assert.assertTrue(verifyAppend(testData, TEST_PATH));
-
- } finally {
- if (appendStream != null) {
- appendStream.close();
- }
- }
- }
-
- /*
- * Test to verify we multiple appends on the same stream.
- */
- @Test
- public void testMultipleAppendsOnSameStream() throws Throwable {
-
- int baseDataSize = 50;
- byte[] baseDataBuffer = createBaseFileWithData(baseDataSize, TEST_PATH);
- int appendDataSize = 100;
- int targetAppendCount = 50;
- byte[] testData = new byte[baseDataSize + (appendDataSize*targetAppendCount)];
- int testDataIndex = 0;
- System.arraycopy(baseDataBuffer, 0, testData, testDataIndex, baseDataSize);
- testDataIndex += baseDataSize;
- int appendCount = 0;
-
- FSDataOutputStream appendStream = null;
-
- try {
-
- while (appendCount < targetAppendCount) {
-
- appendStream = fs.append(TEST_PATH, 50);
-
- int singleAppendChunkSize = 20;
- int appendRunSize = 0;
- while (appendRunSize < appendDataSize) {
-
- byte[] appendDataBuffer = getTestData(singleAppendChunkSize);
- appendStream.write(appendDataBuffer);
- System.arraycopy(appendDataBuffer, 0, testData,
- testDataIndex + appendRunSize, singleAppendChunkSize);
-
- appendRunSize += singleAppendChunkSize;
- }
-
- appendStream.close();
- testDataIndex += appendDataSize;
- appendCount++;
- }
-
- Assert.assertTrue(verifyAppend(testData, TEST_PATH));
- } finally {
- if (appendStream != null) {
- appendStream.close();
- }
- }
- }
-
- @Test(expected=UnsupportedOperationException.class)
- /*
- * Test to verify the behavior when Append Support configuration flag is set to false
- */
- public void testFalseConfigurationFlagBehavior() throws Throwable {
-
- fs = testAccount.getFileSystem();
- Configuration conf = fs.getConf();
- conf.setBoolean(NativeAzureFileSystem.APPEND_SUPPORT_ENABLE_PROPERTY_NAME, false);
- URI uri = fs.getUri();
- fs.initialize(uri, conf);
-
- FSDataOutputStream appendStream = null;
-
- try {
- createBaseFileWithData(0, TEST_PATH);
- appendStream = fs.append(TEST_PATH, 10);
- } finally {
- if (appendStream != null) {
- appendStream.close();
- }
- }
- }
-
- @Override
- protected AzureBlobStorageTestAccount createTestAccount() throws Exception {
- return AzureBlobStorageTestAccount.create();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2d97fa/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAtomicRenameDirList.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAtomicRenameDirList.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAtomicRenameDirList.java
deleted file mode 100644
index 602c1f7..0000000
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAtomicRenameDirList.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import java.io.IOException;
-import java.net.URI;
-
-import org.apache.hadoop.conf.Configuration;
-import org.junit.Test;
-
-public class TestNativeAzureFileSystemAtomicRenameDirList
- extends AbstractWasbTestBase {
- private AzureBlobStorageTestAccount testAccount;
-
- // HBase-site config controlling HBase root dir
- private static final String HBASE_ROOT_DIR_CONF_STRING = "hbase.rootdir";
- private static final String HBASE_ROOT_DIR_VALUE_ON_DIFFERENT_FS = "wasb://somedifferentfilesystem.blob.core.windows.net/hbase";
- @Override
- protected AzureBlobStorageTestAccount createTestAccount() throws Exception {
- testAccount = AzureBlobStorageTestAccount.create();
- return testAccount;
- }
-
- @Test
- public void testAzureNativeStoreIsAtomicRenameKeyDoesNotThrowNPEOnInitializingWithNonDefaultURI () throws IOException {
- NativeAzureFileSystem azureFs = (NativeAzureFileSystem)fs;
- AzureNativeFileSystemStore azureStore = azureFs.getStore();
- Configuration conf = fs.getConf();
- conf.set(HBASE_ROOT_DIR_CONF_STRING, HBASE_ROOT_DIR_VALUE_ON_DIFFERENT_FS);
- URI uri = fs.getUri();
- fs.initialize(uri, conf);
- azureStore.isAtomicRenameKey("anyrandomkey");
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org