You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by tm...@apache.org on 2018/08/08 20:00:50 UTC
[2/4] hadoop git commit: HADOOP-15446. ABFS: tune imports & javadocs;
stabilise tests. Contributed by Steve Loughran and Da Zhou.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/873b519a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java
index d107c9d..d696481 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java
@@ -27,13 +27,11 @@ import org.junit.Test;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
/**
* Test AzureBlobFileSystem back compatibility with WASB.
*/
-public class ITestAzureBlobFileSystemBackCompat extends DependencyInjectedTest {
+public class ITestAzureBlobFileSystemBackCompat extends
+ AbstractAbfsIntegrationTest {
public ITestAzureBlobFileSystemBackCompat() {
super();
}
@@ -54,13 +52,13 @@ public class ITestAzureBlobFileSystemBackCompat extends DependencyInjectedTest {
blockBlob.uploadText("");
FileStatus[] fileStatuses = fs.listStatus(new Path("/test/10/"));
- assertEquals(fileStatuses.length, 2);
- assertEquals(fileStatuses[0].getPath().getName(), "10");
+ assertEquals(2, fileStatuses.length);
+ assertEquals("10", fileStatuses[0].getPath().getName());
assertTrue(fileStatuses[0].isDirectory());
- assertEquals(fileStatuses[0].getLen(), 0);
- assertEquals(fileStatuses[1].getPath().getName(), "123");
+ assertEquals(0, fileStatuses[0].getLen());
+ assertEquals("123", fileStatuses[1].getPath().getName());
assertTrue(fileStatuses[1].isDirectory());
- assertEquals(fileStatuses[1].getLen(), 0);
+ assertEquals(0, fileStatuses[1].getLen());
}
private String getBlobConnectionString() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/873b519a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCopy.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCopy.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCopy.java
index c158e03..90eff97 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCopy.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCopy.java
@@ -33,30 +33,29 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.assertIsFile;
/**
* Test copy operation.
*/
-public class ITestAzureBlobFileSystemCopy extends DependencyInjectedTest {
+public class ITestAzureBlobFileSystemCopy extends AbstractAbfsIntegrationTest {
public ITestAzureBlobFileSystemCopy() {
super();
}
@Test
public void testCopyFromLocalFileSystem() throws Exception {
- final AzureBlobFileSystem fs = this.getFileSystem();
+ final AzureBlobFileSystem fs = getFileSystem();
Path localFilePath = new Path(System.getProperty("test.build.data",
"azure_test"));
- FileSystem localFs = FileSystem.get(new Configuration());
+ FileSystem localFs = FileSystem.getLocal(new Configuration());
localFs.delete(localFilePath, true);
try {
writeString(localFs, localFilePath, "Testing");
Path dstPath = new Path("copiedFromLocal");
assertTrue(FileUtil.copy(localFs, localFilePath, fs, dstPath, false,
fs.getConf()));
- assertTrue(fs.exists(dstPath));
+ assertIsFile(fs, dstPath);
assertEquals("Testing", readString(fs, dstPath));
fs.delete(dstPath, true);
} finally {
@@ -65,36 +64,32 @@ public class ITestAzureBlobFileSystemCopy extends DependencyInjectedTest {
}
private String readString(FileSystem fs, Path testFile) throws IOException {
- FSDataInputStream inputStream = fs.open(testFile);
- String ret = readString(inputStream);
- inputStream.close();
- return ret;
+ return readString(fs.open(testFile));
}
private String readString(FSDataInputStream inputStream) throws IOException {
- BufferedReader reader = new BufferedReader(new InputStreamReader(
- inputStream));
- final int bufferSize = 1024;
- char[] buffer = new char[bufferSize];
- int count = reader.read(buffer, 0, bufferSize);
- if (count > bufferSize) {
- throw new IOException("Exceeded buffer size");
+ try (BufferedReader reader = new BufferedReader(new InputStreamReader(
+ inputStream))) {
+ final int bufferSize = 1024;
+ char[] buffer = new char[bufferSize];
+ int count = reader.read(buffer, 0, bufferSize);
+ if (count > bufferSize) {
+ throw new IOException("Exceeded buffer size");
+ }
+ return new String(buffer, 0, count);
}
- inputStream.close();
- return new String(buffer, 0, count);
}
private void writeString(FileSystem fs, Path path, String value)
throws IOException {
- FSDataOutputStream outputStream = fs.create(path, true);
- writeString(outputStream, value);
+ writeString(fs.create(path, true), value);
}
private void writeString(FSDataOutputStream outputStream, String value)
throws IOException {
- BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(
- outputStream));
- writer.write(value);
- writer.close();
+ try(BufferedWriter writer = new BufferedWriter(
+ new OutputStreamWriter(outputStream))) {
+ writer.write(value);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/873b519a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java
index c9b99e6..1e43f9a 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java
@@ -24,18 +24,17 @@ import java.util.EnumSet;
import org.junit.Test;
import org.apache.hadoop.fs.CreateFlag;
-import org.apache.hadoop.fs.FileAlreadyExistsException;
-import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.assertIsFile;
/**
* Test create operation.
*/
-public class ITestAzureBlobFileSystemCreate extends DependencyInjectedTest {
+public class ITestAzureBlobFileSystemCreate extends
+ AbstractAbfsIntegrationTest {
private static final Path TEST_FILE_PATH = new Path("testfile");
private static final Path TEST_FOLDER_PATH = new Path("testFolder");
private static final String TEST_CHILD_FILE = "childFile";
@@ -43,68 +42,65 @@ public class ITestAzureBlobFileSystemCreate extends DependencyInjectedTest {
super();
}
- @Test(expected = FileAlreadyExistsException.class)
- public void testCreateFileWithExistingDir() throws Exception {
- final AzureBlobFileSystem fs = this.getFileSystem();
- fs.mkdirs(TEST_FOLDER_PATH);
- fs.create(TEST_FOLDER_PATH);
- }
-
@Test
- public void testEnsureFileCreated() throws Exception {
- final AzureBlobFileSystem fs = this.getFileSystem();
- fs.create(TEST_FILE_PATH);
-
- FileStatus fileStatus = fs.getFileStatus(TEST_FILE_PATH);
- assertNotNull(fileStatus);
+ public void testEnsureFileCreatedImmediately() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ FSDataOutputStream out = fs.create(TEST_FILE_PATH);
+ try {
+ assertIsFile(fs, TEST_FILE_PATH);
+ } finally {
+ out.close();
+ }
+ assertIsFile(fs, TEST_FILE_PATH);
}
@Test
@SuppressWarnings("deprecation")
public void testCreateNonRecursive() throws Exception {
- final AzureBlobFileSystem fs = this.getFileSystem();
+ final AzureBlobFileSystem fs = getFileSystem();
Path testFile = new Path(TEST_FOLDER_PATH, TEST_CHILD_FILE);
try {
fs.createNonRecursive(testFile, true, 1024, (short) 1, 1024, null);
- assertTrue("Should've thrown", false);
- } catch (FileNotFoundException e) {
+ fail("Should've thrown");
+ } catch (FileNotFoundException expected) {
}
fs.mkdirs(TEST_FOLDER_PATH);
fs.createNonRecursive(testFile, true, 1024, (short) 1, 1024, null)
.close();
- assertTrue(fs.exists(testFile));
+ assertIsFile(fs, testFile);
}
@Test
@SuppressWarnings("deprecation")
public void testCreateNonRecursive1() throws Exception {
- final AzureBlobFileSystem fs = this.getFileSystem();
+ final AzureBlobFileSystem fs = getFileSystem();
Path testFile = new Path(TEST_FOLDER_PATH, TEST_CHILD_FILE);
try {
fs.createNonRecursive(testFile, FsPermission.getDefault(), EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE), 1024, (short) 1, 1024, null);
- assertTrue("Should've thrown", false);
- } catch (FileNotFoundException e) {
+ fail("Should've thrown");
+ } catch (FileNotFoundException expected) {
}
fs.mkdirs(TEST_FOLDER_PATH);
fs.createNonRecursive(testFile, true, 1024, (short) 1, 1024, null)
.close();
- assertTrue(fs.exists(testFile));
+ assertIsFile(fs, testFile);
+
}
@Test
@SuppressWarnings("deprecation")
public void testCreateNonRecursive2() throws Exception {
- final AzureBlobFileSystem fs = this.getFileSystem();
+ final AzureBlobFileSystem fs = getFileSystem();
Path testFile = new Path(TEST_FOLDER_PATH, TEST_CHILD_FILE);
try {
fs.createNonRecursive(testFile, FsPermission.getDefault(), false, 1024, (short) 1, 1024, null);
- assertTrue("Should've thrown", false);
+ fail("Should've thrown");
} catch (FileNotFoundException e) {
}
fs.mkdirs(TEST_FOLDER_PATH);
fs.createNonRecursive(testFile, true, 1024, (short) 1, 1024, null)
.close();
- assertTrue(fs.exists(testFile));
+ assertIsFile(fs, testFile);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/873b519a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java
index 372a087..91d1723 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java
@@ -28,71 +28,79 @@ import java.util.concurrent.Future;
import org.junit.Test;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
-import static org.junit.Assert.assertEquals;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.assertDeleted;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.assertPathDoesNotExist;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
/**
* Test delete operation.
*/
-public class ITestAzureBlobFileSystemDelete extends DependencyInjectedTest {
+public class ITestAzureBlobFileSystemDelete extends
+ AbstractAbfsIntegrationTest {
public ITestAzureBlobFileSystemDelete() {
super();
}
@Test
public void testDeleteRoot() throws Exception {
- final AzureBlobFileSystem fs = this.getFileSystem();
+ final AzureBlobFileSystem fs = getFileSystem();
fs.mkdirs(new Path("/testFolder0"));
fs.mkdirs(new Path("/testFolder1"));
fs.mkdirs(new Path("/testFolder2"));
- fs.create(new Path("/testFolder1/testfile"));
- fs.create(new Path("/testFolder1/testfile2"));
- fs.create(new Path("/testFolder1/testfile3"));
+ touch(new Path("/testFolder1/testfile"));
+ touch(new Path("/testFolder1/testfile2"));
+ touch(new Path("/testFolder1/testfile3"));
- FileStatus[] ls = fs.listStatus(new Path("/"));
- assertEquals(4, ls.length); // and user dir
+ Path root = new Path("/");
+ FileStatus[] ls = fs.listStatus(root);
+ assertEquals(3, ls.length);
- fs.delete(new Path("/"), true);
- ls = fs.listStatus(new Path("/"));
- assertEquals(0, ls.length);
+ fs.delete(root, true);
+ ls = fs.listStatus(root);
+ assertEquals("listing size", 0, ls.length);
}
- @Test(expected = FileNotFoundException.class)
+ @Test()
public void testOpenFileAfterDelete() throws Exception {
- final AzureBlobFileSystem fs = this.getFileSystem();
- fs.create(new Path("/testFile"));
- fs.delete(new Path("/testFile"), false);
+ final AzureBlobFileSystem fs = getFileSystem();
+ Path testfile = new Path("/testFile");
+ touch(testfile);
+ assertDeleted(fs, testfile, false);
- fs.open(new Path("/testFile"));
+ intercept(FileNotFoundException.class,
+ () -> fs.open(testfile));
}
- @Test(expected = FileNotFoundException.class)
+ @Test
public void testEnsureFileIsDeleted() throws Exception {
- final AzureBlobFileSystem fs = this.getFileSystem();
- fs.create(new Path("testfile"));
- fs.delete(new Path("testfile"), false);
-
- fs.getFileStatus(new Path("testfile"));
+ final AzureBlobFileSystem fs = getFileSystem();
+ Path testfile = new Path("testfile");
+ touch(testfile);
+ assertDeleted(fs, testfile, false);
+ assertPathDoesNotExist(fs, "deleted", testfile);
}
- @Test(expected = FileNotFoundException.class)
+ @Test
public void testDeleteDirectory() throws Exception {
- final AzureBlobFileSystem fs = this.getFileSystem();
- fs.mkdirs(new Path("testfile"));
+ final AzureBlobFileSystem fs = getFileSystem();
+ Path dir = new Path("testfile");
+ fs.mkdirs(dir);
fs.mkdirs(new Path("testfile/test1"));
fs.mkdirs(new Path("testfile/test1/test2"));
- fs.delete(new Path("testfile"), true);
- fs.getFileStatus(new Path("testfile"));
+ assertDeleted(fs, dir, true);
+ assertPathDoesNotExist(fs, "deleted", dir);
}
- @Test(expected = FileNotFoundException.class)
+ @Test
public void testDeleteFirstLevelDirectory() throws Exception {
- final AzureBlobFileSystem fs = this.getFileSystem();
- final List<Future> tasks = new ArrayList<>();
+ final AzureBlobFileSystem fs = getFileSystem();
+ final List<Future<Void>> tasks = new ArrayList<>();
ExecutorService es = Executors.newFixedThreadPool(10);
for (int i = 0; i < 1000; i++) {
@@ -100,7 +108,7 @@ public class ITestAzureBlobFileSystemDelete extends DependencyInjectedTest {
Callable<Void> callable = new Callable<Void>() {
@Override
public Void call() throws Exception {
- fs.create(fileName);
+ touch(fileName);
return null;
}
};
@@ -113,7 +121,12 @@ public class ITestAzureBlobFileSystemDelete extends DependencyInjectedTest {
}
es.shutdownNow();
- fs.delete(new Path("/test"), true);
- fs.getFileStatus(new Path("/test"));
+ Path dir = new Path("/test");
+ // first try a non-recursive delete, expect failure
+ intercept(FileAlreadyExistsException.class,
+ () -> fs.delete(dir, false));
+ assertDeleted(fs, dir, true);
+ assertPathDoesNotExist(fs, "deleted", dir);
+
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/873b519a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2E.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2E.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2E.java
index ad22f99..057dfa0 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2E.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2E.java
@@ -38,7 +38,7 @@ import static org.junit.Assert.assertArrayEquals;
/**
* Test end to end between ABFS client and ABFS server.
*/
-public class ITestAzureBlobFileSystemE2E extends DependencyInjectedTest {
+public class ITestAzureBlobFileSystemE2E extends AbstractAbfsIntegrationTest {
private static final Path TEST_FILE = new Path("testfile");
private static final int TEST_BYTE = 100;
private static final int TEST_OFFSET = 100;
@@ -53,11 +53,11 @@ public class ITestAzureBlobFileSystemE2E extends DependencyInjectedTest {
@Test
public void testWriteOneByteToFile() throws Exception {
- final AzureBlobFileSystem fs = this.getFileSystem();
- FSDataOutputStream stream = fs.create(TEST_FILE);
+ final AzureBlobFileSystem fs = getFileSystem();
- stream.write(TEST_BYTE);
- stream.close();
+ try(FSDataOutputStream stream = fs.create(TEST_FILE)) {
+ stream.write(TEST_BYTE);
+ }
FileStatus fileStatus = fs.getFileStatus(TEST_FILE);
assertEquals(1, fileStatus.getLen());
@@ -65,52 +65,52 @@ public class ITestAzureBlobFileSystemE2E extends DependencyInjectedTest {
@Test
public void testReadWriteBytesToFile() throws Exception {
- final AzureBlobFileSystem fs = this.getFileSystem();
+ final AzureBlobFileSystem fs = getFileSystem();
testWriteOneByteToFile();
- FSDataInputStream inputStream = fs.open(TEST_FILE, TEST_DEFAULT_BUFFER_SIZE);
- int i = inputStream.read();
- inputStream.close();
-
- assertEquals(TEST_BYTE, i);
+ try(FSDataInputStream inputStream = fs.open(TEST_FILE,
+ TEST_DEFAULT_BUFFER_SIZE)) {
+ assertEquals(TEST_BYTE, inputStream.read());
+ }
}
@Test (expected = IOException.class)
public void testOOBWrites() throws Exception {
- final AzureBlobFileSystem fs = this.getFileSystem();
+ final AzureBlobFileSystem fs = getFileSystem();
int readBufferSize = fs.getAbfsStore().getAbfsConfiguration().getReadBufferSize();
- fs.create(TEST_FILE);
- FSDataOutputStream writeStream = fs.create(TEST_FILE);
-
byte[] bytesToRead = new byte[readBufferSize];
final byte[] b = new byte[2 * readBufferSize];
new Random().nextBytes(b);
- writeStream.write(b);
- writeStream.flush();
- writeStream.close();
- FSDataInputStream readStream = fs.open(TEST_FILE);
- readStream.read(bytesToRead, 0, readBufferSize);
+ try(FSDataOutputStream writeStream = fs.create(TEST_FILE)) {
+ writeStream.write(b);
+ writeStream.flush();
+ }
+
+ try (FSDataInputStream readStream = fs.open(TEST_FILE)) {
+ assertEquals(readBufferSize,
+ readStream.read(bytesToRead, 0, readBufferSize));
- writeStream = fs.create(TEST_FILE);
- writeStream.write(b);
- writeStream.flush();
- writeStream.close();
+ try (FSDataOutputStream writeStream = fs.create(TEST_FILE)) {
+ writeStream.write(b);
+ writeStream.flush();
+ }
- readStream.read(bytesToRead, 0, readBufferSize);
- readStream.close();
+ assertEquals(readBufferSize,
+ readStream.read(bytesToRead, 0, readBufferSize));
+ }
}
@Test
public void testWriteWithBufferOffset() throws Exception {
- final AzureBlobFileSystem fs = this.getFileSystem();
- final FSDataOutputStream stream = fs.create(TEST_FILE);
+ final AzureBlobFileSystem fs = getFileSystem();
final byte[] b = new byte[1024 * 1000];
new Random().nextBytes(b);
- stream.write(b, TEST_OFFSET, b.length - TEST_OFFSET);
- stream.close();
+ try(final FSDataOutputStream stream = fs.create(TEST_FILE)) {
+ stream.write(b, TEST_OFFSET, b.length - TEST_OFFSET);
+ }
final byte[] r = new byte[TEST_DEFAULT_READ_BUFFER_SIZE];
FSDataInputStream inputStream = fs.open(TEST_FILE, TEST_DEFAULT_BUFFER_SIZE);
@@ -124,13 +124,11 @@ public class ITestAzureBlobFileSystemE2E extends DependencyInjectedTest {
@Test
public void testReadWriteHeavyBytesToFileWithSmallerChunks() throws Exception {
- final AzureBlobFileSystem fs = this.getFileSystem();
- final FSDataOutputStream stream = fs.create(TEST_FILE);
+ final AzureBlobFileSystem fs = getFileSystem();
final byte[] writeBuffer = new byte[5 * 1000 * 1024];
new Random().nextBytes(writeBuffer);
- stream.write(writeBuffer);
- stream.close();
+ write(TEST_FILE, writeBuffer);
final byte[] readBuffer = new byte[5 * 1000 * 1024];
FSDataInputStream inputStream = fs.open(TEST_FILE, TEST_DEFAULT_BUFFER_SIZE);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/873b519a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2EScale.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2EScale.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2EScale.java
index 616253b..04690de 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2EScale.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2EScale.java
@@ -26,7 +26,6 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
-import org.junit.Assert;
import org.junit.Test;
import org.apache.hadoop.fs.FSDataInputStream;
@@ -35,28 +34,24 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotEquals;
-
/**
* Test end to end between ABFS client and ABFS server with heavy traffic.
*/
-public class ITestAzureBlobFileSystemE2EScale extends DependencyInjectedTest {
+public class ITestAzureBlobFileSystemE2EScale extends
+ AbstractAbfsScaleTest {
private static final int TEN = 10;
private static final int ONE_THOUSAND = 1000;
private static final int BASE_SIZE = 1024;
private static final int ONE_MB = 1024 * 1024;
private static final int DEFAULT_WRITE_TIMES = 100;
- private static final Path TEST_FILE = new Path("testfile");
+ private static final Path TEST_FILE = new Path("ITestAzureBlobFileSystemE2EScale");
public ITestAzureBlobFileSystemE2EScale() {
- super();
}
@Test
- public void testWriteHeavyBytesToFile() throws Exception {
- final AzureBlobFileSystem fs = this.getFileSystem();
+ public void testWriteHeavyBytesToFileAcrossThreads() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
final FSDataOutputStream stream = fs.create(TEST_FILE);
ExecutorService es = Executors.newFixedThreadPool(TEN);
@@ -65,7 +60,8 @@ public class ITestAzureBlobFileSystemE2EScale extends DependencyInjectedTest {
new Random().nextBytes(b);
List<Future<Void>> tasks = new ArrayList<>();
- for (int i = 0; i < DEFAULT_WRITE_TIMES; i++) {
+ int operationCount = DEFAULT_WRITE_TIMES;
+ for (int i = 0; i < operationCount; i++) {
Callable<Void> callable = new Callable<Void>() {
@Override
public Void call() throws Exception {
@@ -86,48 +82,38 @@ public class ITestAzureBlobFileSystemE2EScale extends DependencyInjectedTest {
es.shutdownNow();
FileStatus fileStatus = fs.getFileStatus(TEST_FILE);
- assertEquals(testWriteBufferSize * DEFAULT_WRITE_TIMES, fileStatus.getLen());
- }
-
- @Test
- public void testReadWriteHeavyBytesToFile() throws Exception {
- final AzureBlobFileSystem fs = this.getFileSystem();
- final FSDataOutputStream stream = fs.create(TEST_FILE);
-
- int testBufferSize = 5 * TEN * ONE_THOUSAND * BASE_SIZE;
- final byte[] b = new byte[testBufferSize];
- new Random().nextBytes(b);
- stream.write(b);
- stream.close();
-
- final byte[] r = new byte[testBufferSize];
- FSDataInputStream inputStream = fs.open(TEST_FILE, 4 * ONE_MB);
- int result = inputStream.read(r);
- inputStream.close();
-
- assertNotEquals(-1, result);
- assertArrayEquals(r, b);
+ assertEquals(testWriteBufferSize * operationCount, fileStatus.getLen());
}
@Test
public void testReadWriteHeavyBytesToFileWithStatistics() throws Exception {
- final AzureBlobFileSystem fs = this.getFileSystem();
- final FSDataOutputStream stream = fs.create(TEST_FILE);
- final FileSystem.Statistics abfsStatistics = fs.getFsStatistics();
- abfsStatistics.reset();
+ final AzureBlobFileSystem fs = getFileSystem();
+ final FileSystem.Statistics abfsStatistics;
+ int testBufferSize;
+ final byte[] sourceData;
+ try(final FSDataOutputStream stream = fs.create(TEST_FILE)) {
+ abfsStatistics = fs.getFsStatistics();
+ abfsStatistics.reset();
+
+ testBufferSize = 5 * TEN * ONE_THOUSAND * BASE_SIZE;
+ sourceData = new byte[testBufferSize];
+ new Random().nextBytes(sourceData);
+ stream.write(sourceData);
+ }
- int testBufferSize = 5 * TEN * ONE_THOUSAND * BASE_SIZE;
- final byte[] b = new byte[testBufferSize];
- new Random().nextBytes(b);
- stream.write(b);
- stream.close();
+ final byte[] remoteData = new byte[testBufferSize];
+ int bytesRead;
+ try (FSDataInputStream inputStream = fs.open(TEST_FILE, 4 * ONE_MB)) {
+ bytesRead = inputStream.read(remoteData);
+ }
- final byte[] r = new byte[testBufferSize];
- FSDataInputStream inputStream = fs.open(TEST_FILE, 4 * ONE_MB);
- inputStream.read(r);
- inputStream.close();
+ String stats = abfsStatistics.toString();
+ assertEquals("Bytes read in " + stats,
+ remoteData.length, abfsStatistics.getBytesRead());
+ assertEquals("bytes written in " + stats,
+ sourceData.length, abfsStatistics.getBytesWritten());
+ assertEquals("bytesRead from read() call", testBufferSize, bytesRead );
+ assertArrayEquals("round tripped data", sourceData, remoteData);
- Assert.assertEquals(r.length, abfsStatistics.getBytesRead());
- Assert.assertEquals(b.length, abfsStatistics.getBytesWritten());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/873b519a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java
index bfa662d..791694b 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.fs.azurebfs;
+import java.io.IOException;
+
import org.junit.Test;
import org.apache.hadoop.fs.FileStatus;
@@ -25,12 +27,11 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
-import static org.junit.Assert.assertEquals;
-
/**
* Test FileStatus.
*/
-public class ITestAzureBlobFileSystemFileStatus extends DependencyInjectedTest {
+public class ITestAzureBlobFileSystemFileStatus extends
+ AbstractAbfsIntegrationTest {
private static final Path TEST_FILE = new Path("testFile");
private static final Path TEST_FOLDER = new Path("testDir");
public ITestAzureBlobFileSystemFileStatus() {
@@ -41,24 +42,38 @@ public class ITestAzureBlobFileSystemFileStatus extends DependencyInjectedTest {
public void testEnsureStatusWorksForRoot() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- fs.getFileStatus(new Path("/"));
- fs.listStatus(new Path("/"));
+ Path root = new Path("/");
+ FileStatus[] rootls = fs.listStatus(root);
+ assertEquals("root listing", 0, rootls.length);
}
@Test
public void testFileStatusPermissionsAndOwnerAndGroup() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- fs.create(TEST_FILE);
- fs.mkdirs(TEST_FOLDER);
+ touch(TEST_FILE);
+ validateStatus(fs, TEST_FILE);
+ }
+
+ private FileStatus validateStatus(final AzureBlobFileSystem fs, final Path name)
+ throws IOException {
+ FileStatus fileStatus = fs.getFileStatus(name);
+ String errorInStatus = "error in " + fileStatus + " from " + fs;
+ assertEquals(errorInStatus + ": permission",
+ new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL),
+ fileStatus.getPermission());
+ assertEquals(errorInStatus + ": owner",
+ fs.getOwnerUser(), fileStatus.getOwner());
+ assertEquals(errorInStatus + ": group",
+ fs.getOwnerUserPrimaryGroup(), fileStatus.getGroup());
+ return fileStatus;
+ }
- FileStatus fileStatus = fs.getFileStatus(TEST_FILE);
- assertEquals(new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL), fileStatus.getPermission());
- assertEquals(fs.getOwnerUser(), fileStatus.getGroup());
- assertEquals(fs.getOwnerUserPrimaryGroup(), fileStatus.getOwner());
+ @Test
+ public void testFolderStatusPermissionsAndOwnerAndGroup() throws Exception {
+ final AzureBlobFileSystem fs = this.getFileSystem();
+ fs.mkdirs(TEST_FOLDER);
- fileStatus = fs.getFileStatus(TEST_FOLDER);
- assertEquals(new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL), fileStatus.getPermission());
- assertEquals(fs.getOwnerUser(), fileStatus.getGroup());
- assertEquals(fs.getOwnerUserPrimaryGroup(), fileStatus.getOwner());
+ validateStatus(fs, TEST_FOLDER);
}
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/873b519a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFlush.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFlush.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFlush.java
index 8c2e8ce..d90f018 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFlush.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFlush.java
@@ -34,14 +34,10 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotEquals;
-
/**
* Test flush operation.
*/
-public class ITestAzureBlobFileSystemFlush extends DependencyInjectedTest {
+public class ITestAzureBlobFileSystemFlush extends AbstractAbfsScaleTest {
private static final int BASE_SIZE = 1024;
private static final int ONE_THOUSAND = 1000;
private static final int TEST_BUFFER_SIZE = 5 * ONE_THOUSAND * BASE_SIZE;
@@ -56,146 +52,145 @@ public class ITestAzureBlobFileSystemFlush extends DependencyInjectedTest {
}
@Test
- public void testAbfsOutputStreamAsyncFlushWithRetainUncommitedData() throws Exception {
- final AzureBlobFileSystem fs = this.getFileSystem();
- final FSDataOutputStream stream = fs.create(TEST_FILE_PATH);
-
- final byte[] b = new byte[TEST_BUFFER_SIZE];
- new Random().nextBytes(b);
+ public void testAbfsOutputStreamAsyncFlushWithRetainUncommittedData() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ final byte[] b;
+ try(final FSDataOutputStream stream = fs.create(TEST_FILE_PATH)) {
+ b = new byte[TEST_BUFFER_SIZE];
+ new Random().nextBytes(b);
- for (int i = 0; i < 2; i++) {
- stream.write(b);
+ for (int i = 0; i < 2; i++) {
+ stream.write(b);
- for (int j = 0; j < FLUSH_TIMES; j++) {
- stream.flush();
- Thread.sleep(10);
+ for (int j = 0; j < FLUSH_TIMES; j++) {
+ stream.flush();
+ Thread.sleep(10);
+ }
}
}
- stream.close();
-
final byte[] r = new byte[TEST_BUFFER_SIZE];
- FSDataInputStream inputStream = fs.open(TEST_FILE_PATH, 4 * ONE_MB);
+ try(FSDataInputStream inputStream = fs.open(TEST_FILE_PATH, 4 * ONE_MB)) {
+ while (inputStream.available() != 0) {
+ int result = inputStream.read(r);
- while (inputStream.available() != 0) {
- int result = inputStream.read(r);
-
- assertNotEquals(-1, result);
- assertArrayEquals(r, b);
+ assertNotEquals("read returned -1", -1, result);
+ assertArrayEquals("buffer read from stream", r, b);
+ }
}
-
- inputStream.close();
}
@Test
public void testAbfsOutputStreamSyncFlush() throws Exception {
- final AzureBlobFileSystem fs = this.getFileSystem();
- final FSDataOutputStream stream = fs.create(TEST_FILE_PATH);
-
- final byte[] b = new byte[TEST_BUFFER_SIZE];
- new Random().nextBytes(b);
- stream.write(b);
+ final AzureBlobFileSystem fs = getFileSystem();
+ final byte[] b;
+ try(final FSDataOutputStream stream = fs.create(TEST_FILE_PATH)) {
+ b = new byte[TEST_BUFFER_SIZE];
+ new Random().nextBytes(b);
+ stream.write(b);
- for (int i = 0; i < FLUSH_TIMES; i++) {
- stream.hsync();
- stream.hflush();
- Thread.sleep(10);
+ for (int i = 0; i < FLUSH_TIMES; i++) {
+ stream.hsync();
+ stream.hflush();
+ Thread.sleep(10);
+ }
}
- stream.close();
final byte[] r = new byte[TEST_BUFFER_SIZE];
- FSDataInputStream inputStream = fs.open(TEST_FILE_PATH, 4 * ONE_MB);
- int result = inputStream.read(r);
-
- assertNotEquals(-1, result);
- assertArrayEquals(r, b);
+ try(FSDataInputStream inputStream = fs.open(TEST_FILE_PATH, 4 * ONE_MB)) {
+ int result = inputStream.read(r);
- inputStream.close();
+ assertNotEquals(-1, result);
+ assertArrayEquals(r, b);
+ }
}
@Test
public void testWriteHeavyBytesToFileSyncFlush() throws Exception {
- final AzureBlobFileSystem fs = this.getFileSystem();
- final FSDataOutputStream stream = fs.create(TEST_FILE_PATH);
- final FileSystem.Statistics abfsStatistics = fs.getFsStatistics();
- abfsStatistics.reset();
-
- ExecutorService es = Executors.newFixedThreadPool(10);
-
- final byte[] b = new byte[TEST_BUFFER_SIZE];
- new Random().nextBytes(b);
-
- List<Future<Void>> tasks = new ArrayList<>();
- for (int i = 0; i < FLUSH_TIMES; i++) {
- Callable<Void> callable = new Callable<Void>() {
- @Override
- public Void call() throws Exception {
- stream.write(b);
- return null;
- }
- };
-
- tasks.add(es.submit(callable));
- }
+ final AzureBlobFileSystem fs = getFileSystem();
+ final FileSystem.Statistics abfsStatistics;
+ ExecutorService es;
+ try(final FSDataOutputStream stream = fs.create(TEST_FILE_PATH)) {
+ abfsStatistics = fs.getFsStatistics();
+ abfsStatistics.reset();
+
+ es = Executors.newFixedThreadPool(10);
+
+ final byte[] b = new byte[TEST_BUFFER_SIZE];
+ new Random().nextBytes(b);
+
+ List<Future<Void>> tasks = new ArrayList<>();
+ for (int i = 0; i < FLUSH_TIMES; i++) {
+ Callable<Void> callable = new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ stream.write(b);
+ return null;
+ }
+ };
+
+ tasks.add(es.submit(callable));
+ }
- boolean shouldStop = false;
- while (!shouldStop) {
- shouldStop = true;
- for (Future<Void> task : tasks) {
- if (!task.isDone()) {
- stream.hsync();
- shouldStop = false;
- Thread.sleep(THREAD_SLEEP_TIME);
+ boolean shouldStop = false;
+ while (!shouldStop) {
+ shouldStop = true;
+ for (Future<Void> task : tasks) {
+ if (!task.isDone()) {
+ stream.hsync();
+ shouldStop = false;
+ Thread.sleep(THREAD_SLEEP_TIME);
+ }
}
}
- }
- tasks.clear();
- stream.close();
+ tasks.clear();
+ }
es.shutdownNow();
FileStatus fileStatus = fs.getFileStatus(TEST_FILE_PATH);
- assertEquals((long) TEST_BUFFER_SIZE * FLUSH_TIMES, fileStatus.getLen());
- assertEquals((long) TEST_BUFFER_SIZE * FLUSH_TIMES, abfsStatistics.getBytesWritten());
+ long expectedWrites = (long) TEST_BUFFER_SIZE * FLUSH_TIMES;
+ assertEquals("Wrong file length in " + fileStatus, expectedWrites, fileStatus.getLen());
+ assertEquals("wrong bytes Written count in " + abfsStatistics,
+ expectedWrites, abfsStatistics.getBytesWritten());
}
@Test
public void testWriteHeavyBytesToFileAsyncFlush() throws Exception {
- final AzureBlobFileSystem fs = this.getFileSystem();
- fs.create(TEST_FILE_PATH);
- final FSDataOutputStream stream = fs.create(TEST_FILE_PATH);
+ final AzureBlobFileSystem fs = getFileSystem();
ExecutorService es = Executors.newFixedThreadPool(10);
+ try(final FSDataOutputStream stream = fs.create(TEST_FILE_PATH)) {
+
+ final byte[] b = new byte[TEST_BUFFER_SIZE];
+ new Random().nextBytes(b);
+
+ List<Future<Void>> tasks = new ArrayList<>();
+ for (int i = 0; i < FLUSH_TIMES; i++) {
+ Callable<Void> callable = new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ stream.write(b);
+ return null;
+ }
+ };
+
+ tasks.add(es.submit(callable));
+ }
- final byte[] b = new byte[TEST_BUFFER_SIZE];
- new Random().nextBytes(b);
-
- List<Future<Void>> tasks = new ArrayList<>();
- for (int i = 0; i < FLUSH_TIMES; i++) {
- Callable<Void> callable = new Callable<Void>() {
- @Override
- public Void call() throws Exception {
- stream.write(b);
- return null;
- }
- };
-
- tasks.add(es.submit(callable));
- }
-
- boolean shouldStop = false;
- while (!shouldStop) {
- shouldStop = true;
- for (Future<Void> task : tasks) {
- if (!task.isDone()) {
- stream.flush();
- shouldStop = false;
+ boolean shouldStop = false;
+ while (!shouldStop) {
+ shouldStop = true;
+ for (Future<Void> task : tasks) {
+ if (!task.isDone()) {
+ stream.flush();
+ shouldStop = false;
+ }
}
}
+ Thread.sleep(THREAD_SLEEP_TIME);
+ tasks.clear();
}
- Thread.sleep(THREAD_SLEEP_TIME);
- tasks.clear();
- stream.close();
es.shutdownNow();
FileStatus fileStatus = fs.getFileStatus(TEST_FILE_PATH);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/873b519a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemInitAndCreate.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemInitAndCreate.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemInitAndCreate.java
index d2ed400..5a6e46d 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemInitAndCreate.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemInitAndCreate.java
@@ -22,29 +22,32 @@ import java.io.FileNotFoundException;
import org.junit.Test;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys;
/**
* Test filesystem initialization and creation.
*/
-public class ITestAzureBlobFileSystemInitAndCreate extends DependencyInjectedTest {
+public class ITestAzureBlobFileSystemInitAndCreate extends
+ AbstractAbfsIntegrationTest {
public ITestAzureBlobFileSystemInitAndCreate() {
- super();
this.getConfiguration().unset(ConfigurationKeys.AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION);
}
@Override
- public void initialize() {
+ public void setup() {
}
@Override
- public void testCleanup() {
+ public void teardown() {
}
@Test (expected = FileNotFoundException.class)
public void ensureFilesystemWillNotBeCreatedIfCreationConfigIsNotSet() throws Exception {
- super.initialize();
- this.getFileSystem();
+ super.setup();
+ final AzureBlobFileSystem fs = this.getFileSystem();
+ FileStatus[] fileStatuses = fs.listStatus(new Path("/"));
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/873b519a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemListStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemListStatus.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemListStatus.java
index 6059766..b87abe6 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemListStatus.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemListStatus.java
@@ -26,20 +26,21 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
-import org.junit.Assert;
import org.junit.Test;
+import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.assertFalse;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
/**
* Test listStatus operation.
*/
-public class ITestAzureBlobFileSystemListStatus extends DependencyInjectedTest {
+public class ITestAzureBlobFileSystemListStatus extends
+ AbstractAbfsIntegrationTest {
private static final int TEST_FILES_NUMBER = 6000;
public ITestAzureBlobFileSystemListStatus() {
super();
@@ -47,8 +48,8 @@ public class ITestAzureBlobFileSystemListStatus extends DependencyInjectedTest {
@Test
public void testListPath() throws Exception {
- final AzureBlobFileSystem fs = this.getFileSystem();
- final List<Future> tasks = new ArrayList<>();
+ final AzureBlobFileSystem fs = getFileSystem();
+ final List<Future<Void>> tasks = new ArrayList<>();
ExecutorService es = Executors.newFixedThreadPool(10);
for (int i = 0; i < TEST_FILES_NUMBER; i++) {
@@ -56,7 +57,7 @@ public class ITestAzureBlobFileSystemListStatus extends DependencyInjectedTest {
Callable<Void> callable = new Callable<Void>() {
@Override
public Void call() throws Exception {
- fs.create(fileName);
+ touch(fileName);
return null;
}
};
@@ -70,63 +71,101 @@ public class ITestAzureBlobFileSystemListStatus extends DependencyInjectedTest {
es.shutdownNow();
FileStatus[] files = fs.listStatus(new Path("/"));
- Assert.assertEquals(files.length, TEST_FILES_NUMBER + 1 /* user directory */);
+ assertEquals(TEST_FILES_NUMBER, files.length /* user directory */);
}
+ /**
+ * Creates a file, verifies that listStatus returns it,
+ * even while the file is still open for writing.
+ */
@Test
public void testListFileVsListDir() throws Exception {
- final AzureBlobFileSystem fs = this.getFileSystem();
- fs.create(new Path("/testFile"));
-
- FileStatus[] testFiles = fs.listStatus(new Path("/testFile"));
- Assert.assertEquals(testFiles.length, 1);
- Assert.assertFalse(testFiles[0].isDirectory());
+ final AzureBlobFileSystem fs = getFileSystem();
+ Path path = new Path("/testFile");
+ try(FSDataOutputStream ignored = fs.create(path)) {
+ FileStatus[] testFiles = fs.listStatus(path);
+ assertEquals("length of test files", 1, testFiles.length);
+ FileStatus status = testFiles[0];
+ assertIsFileReference(status);
+ }
}
@Test
public void testListFileVsListDir2() throws Exception {
- final AzureBlobFileSystem fs = this.getFileSystem();
+ final AzureBlobFileSystem fs = getFileSystem();
fs.mkdirs(new Path("/testFolder"));
fs.mkdirs(new Path("/testFolder/testFolder2"));
fs.mkdirs(new Path("/testFolder/testFolder2/testFolder3"));
- fs.create(new Path("/testFolder/testFolder2/testFolder3/testFile"));
-
- FileStatus[] testFiles = fs.listStatus(new Path("/testFolder/testFolder2/testFolder3/testFile"));
- Assert.assertEquals(testFiles.length, 1);
- Assert.assertEquals(testFiles[0].getPath(), new Path(this.getTestUrl(),
- "/testFolder/testFolder2/testFolder3/testFile"));
- Assert.assertFalse(testFiles[0].isDirectory());
+ Path testFile0Path = new Path("/testFolder/testFolder2/testFolder3/testFile");
+ ContractTestUtils.touch(fs, testFile0Path);
+
+ FileStatus[] testFiles = fs.listStatus(testFile0Path);
+ assertEquals("Wrong listing size of file " + testFile0Path,
+ 1, testFiles.length);
+ FileStatus file0 = testFiles[0];
+ assertEquals("Wrong path for " + file0,
+ new Path(getTestUrl(), "/testFolder/testFolder2/testFolder3/testFile"),
+ file0.getPath());
+ assertIsFileReference(file0);
}
@Test(expected = FileNotFoundException.class)
public void testListNonExistentDir() throws Exception {
- final AzureBlobFileSystem fs = this.getFileSystem();
+ final AzureBlobFileSystem fs = getFileSystem();
fs.listStatus(new Path("/testFile/"));
}
@Test
public void testListFiles() throws Exception {
- final AzureBlobFileSystem fs = this.getFileSystem();
- fs.mkdirs(new Path("/test"));
+ final AzureBlobFileSystem fs = getFileSystem();
+ Path testDir = new Path("/test");
+ fs.mkdirs(testDir);
FileStatus[] fileStatuses = fs.listStatus(new Path("/"));
- assertEquals(fileStatuses.length, 2);
+ assertEquals(1, fileStatuses.length);
fs.mkdirs(new Path("/test/sub"));
- fileStatuses = fs.listStatus(new Path("/test"));
- assertEquals(fileStatuses.length, 1);
- assertEquals(fileStatuses[0].getPath().getName(), "sub");
- assertTrue(fileStatuses[0].isDirectory());
- assertEquals(fileStatuses[0].getLen(), 0);
-
- fs.create(new Path("/test/f"));
- fileStatuses = fs.listStatus(new Path("/test"));
- assertEquals(fileStatuses.length, 2);
- assertEquals(fileStatuses[0].getPath().getName(), "f");
- assertFalse(fileStatuses[0].isDirectory());
- assertEquals(fileStatuses[0].getLen(), 0);
- assertEquals(fileStatuses[1].getPath().getName(), "sub");
- assertTrue(fileStatuses[1].isDirectory());
- assertEquals(fileStatuses[1].getLen(), 0);
+ fileStatuses = fs.listStatus(testDir);
+ assertEquals(1, fileStatuses.length);
+ assertEquals("sub", fileStatuses[0].getPath().getName());
+ assertIsDirectoryReference(fileStatuses[0]);
+ Path childF = fs.makeQualified(new Path("/test/f"));
+ touch(childF);
+ fileStatuses = fs.listStatus(testDir);
+ assertEquals(2, fileStatuses.length);
+ final FileStatus childStatus = fileStatuses[0];
+ assertEquals(childF, childStatus.getPath());
+ assertEquals("f", childStatus.getPath().getName());
+ assertIsFileReference(childStatus);
+ assertEquals(0, childStatus.getLen());
+ final FileStatus status1 = fileStatuses[1];
+ assertEquals("sub", status1.getPath().getName());
+ assertIsDirectoryReference(status1);
+ // look at the child through getFileStatus
+ LocatedFileStatus locatedChildStatus = fs.listFiles(childF, false).next();
+ assertIsFileReference(locatedChildStatus);
+
+ fs.delete(testDir, true);
+ intercept(FileNotFoundException.class,
+ () -> fs.listFiles(childF, false).next());
+
+ // do some final checks on the status (failing due to version checks)
+ assertEquals("Path mismatch of " + locatedChildStatus,
+ childF, locatedChildStatus.getPath());
+ assertEquals("locatedstatus.equals(status)",
+ locatedChildStatus, childStatus);
+ assertEquals("status.equals(locatedstatus)",
+ childStatus, locatedChildStatus);
+ }
+
+ private void assertIsDirectoryReference(FileStatus status) {
+ assertTrue("Not a directory: " + status, status.isDirectory());
+ assertFalse("Not a directory: " + status, status.isFile());
+ assertEquals(0, status.getLen());
+ }
+
+ private void assertIsFileReference(FileStatus status) {
+ assertFalse("Not a file: " + status, status.isDirectory());
+ assertTrue("Not a file: " + status, status.isFile());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/873b519a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemMkDir.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemMkDir.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemMkDir.java
index b61908c..1bb2c54 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemMkDir.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemMkDir.java
@@ -18,71 +18,30 @@
package org.apache.hadoop.fs.azurebfs;
-import java.util.concurrent.Callable;
-
import org.junit.Test;
-import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.Path;
-import static org.apache.hadoop.test.LambdaTestUtils.intercept;
-import static org.junit.Assert.assertTrue;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.assertMkdirs;
/**
* Test mkdir operation.
*/
-public class ITestAzureBlobFileSystemMkDir extends DependencyInjectedTest {
+public class ITestAzureBlobFileSystemMkDir extends AbstractAbfsIntegrationTest {
public ITestAzureBlobFileSystemMkDir() {
super();
}
@Test
public void testCreateDirWithExistingDir() throws Exception {
- final AzureBlobFileSystem fs = this.getFileSystem();
- assertTrue(fs.mkdirs(new Path("testFolder")));
- assertTrue(fs.mkdirs(new Path("testFolder")));
- }
-
- @Test(expected = FileAlreadyExistsException.class)
- public void createDirectoryUnderFile() throws Exception {
- final AzureBlobFileSystem fs = this.getFileSystem();
- fs.create(new Path("testFile"));
- fs.mkdirs(new Path("testFile/TestDirectory"));
- }
-
- @Test
- public void testCreateDirectoryOverExistingFiles() throws Exception {
- final AzureBlobFileSystem fs = this.getFileSystem();
- fs.create(new Path("/testPath"));
- FileAlreadyExistsException ex = intercept(
- FileAlreadyExistsException.class,
- new Callable<Void>() {
- @Override
- public Void call() throws Exception {
- fs.mkdirs(new Path("/testPath"));
- return null;
- }
- });
-
- assertTrue(ex instanceof FileAlreadyExistsException);
-
- fs.create(new Path("/testPath1/file1"));
- ex = intercept(
- FileAlreadyExistsException.class,
- new Callable<Void>() {
- @Override
- public Void call() throws Exception {
- fs.mkdirs(new Path("/testPath1/file1"));
- return null;
- }
- });
-
- assertTrue(ex instanceof FileAlreadyExistsException);
+ final AzureBlobFileSystem fs = getFileSystem();
+ Path path = new Path("testFolder");
+ assertMkdirs(fs, path);
+ assertMkdirs(fs, path);
}
@Test
public void testCreateRoot() throws Exception {
- final AzureBlobFileSystem fs = this.getFileSystem();
- assertTrue(fs.mkdirs(new Path("/")));
+ assertMkdirs(getFileSystem(), new Path("/"));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/873b519a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemOpen.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemOpen.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemOpen.java
deleted file mode 100644
index fef7f47..0000000
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemOpen.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azurebfs;
-
-import java.io.FileNotFoundException;
-
-import org.junit.Test;
-
-import org.apache.hadoop.fs.Path;
-
-/**
- * Test open operation.
- */
-public class ITestAzureBlobFileSystemOpen extends DependencyInjectedTest {
- public ITestAzureBlobFileSystemOpen() throws Exception {
- super();
- }
-
- @Test(expected = FileNotFoundException.class)
- public void testOpenDirectory() throws Exception {
- final AzureBlobFileSystem fs = this.getFileSystem();
- fs.mkdirs(new Path("testFolder"));
- fs.open(new Path("testFolder"));
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/873b519a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRandomRead.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRandomRead.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRandomRead.java
index 8b96c69..c61de67 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRandomRead.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRandomRead.java
@@ -18,32 +18,31 @@
package org.apache.hadoop.fs.azurebfs;
-import org.apache.hadoop.fs.Path;
+import java.io.EOFException;
+import java.io.IOException;
+import java.util.Random;
+import java.util.concurrent.Callable;
+
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FSExceptionMessages;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.azure.NativeAzureFileSystem;
import org.apache.hadoop.fs.contract.ContractTestUtils;
-import org.junit.Test;
-
-import java.io.EOFException;
-import java.io.IOException;
-import java.util.Random;
-import java.util.concurrent.Callable;
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertArrayEquals;
-
/**
* Test random read operation.
*/
-public class ITestAzureBlobFileSystemRandomRead extends DependencyInjectedTest {
+public class ITestAzureBlobFileSystemRandomRead extends
+ AbstractAbfsScaleTest {
private static final int KILOBYTE = 1024;
private static final int MEGABYTE = KILOBYTE * KILOBYTE;
private static final long TEST_FILE_SIZE = 8 * MEGABYTE;
@@ -62,6 +61,9 @@ public class ITestAzureBlobFileSystemRandomRead extends DependencyInjectedTest {
private static final String ABFS = "ABFS";
private static long testFileLength = 0;
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ITestAzureBlobFileSystemRandomRead.class);
+
public ITestAzureBlobFileSystemRandomRead() throws Exception {
super();
}
@@ -76,7 +78,7 @@ public class ITestAzureBlobFileSystemRandomRead extends DependencyInjectedTest {
// forward seek and read a kilobyte into first kilobyte of bufferV2
inputStream.seek(5 * MEGABYTE);
int numBytesRead = inputStream.read(buffer, 0, KILOBYTE);
- assertEquals(KILOBYTE, numBytesRead);
+ assertEquals("Wrong number of bytes read", KILOBYTE, numBytesRead);
int len = MEGABYTE;
int offset = buffer.length - len;
@@ -84,7 +86,7 @@ public class ITestAzureBlobFileSystemRandomRead extends DependencyInjectedTest {
// reverse seek and read a megabyte into last megabyte of bufferV1
inputStream.seek(3 * MEGABYTE);
numBytesRead = inputStream.read(buffer, offset, len);
- assertEquals(len, numBytesRead);
+ assertEquals("Wrong number of bytes read after seek", len, numBytesRead);
}
}
@@ -391,7 +393,7 @@ public class ITestAzureBlobFileSystemRandomRead extends DependencyInjectedTest {
afterSeekElapsedMs = sequentialRead(ABFS,
this.getFileSystem(), true);
ratio = afterSeekElapsedMs / beforeSeekElapsedMs;
- System.out.println((String.format(
+ LOG.info((String.format(
"beforeSeekElapsedMs=%1$d, afterSeekElapsedMs=%2$d, ratio=%3$.2f",
(long) beforeSeekElapsedMs,
(long) afterSeekElapsedMs,
@@ -425,7 +427,7 @@ public class ITestAzureBlobFileSystemRandomRead extends DependencyInjectedTest {
ratio = v2ElapsedMs / v1ElapsedMs;
- System.out.println(String.format(
+ LOG.info(String.format(
"v1ElapsedMs=%1$d, v2ElapsedMs=%2$d, ratio=%3$.2f",
(long) v1ElapsedMs,
(long) v2ElapsedMs,
@@ -464,7 +466,7 @@ public class ITestAzureBlobFileSystemRandomRead extends DependencyInjectedTest {
}
long elapsedTimeMs = timer.elapsedTimeMs();
- System.out.println(String.format(
+ LOG.info(String.format(
"v%1$s: bytesRead=%2$d, elapsedMs=%3$d, Mbps=%4$.2f,"
+ " afterReverseSeek=%5$s",
version,
@@ -496,7 +498,7 @@ public class ITestAzureBlobFileSystemRandomRead extends DependencyInjectedTest {
} while (bytesRead > 0 && totalBytesRead < minBytesToRead);
long elapsedTimeMs = timer.elapsedTimeMs();
inputStream.close();
- System.out.println(String.format(
+ LOG.info(String.format(
"v%1$d: totalBytesRead=%2$d, elapsedTimeMs=%3$d, Mbps=%4$.2f",
version,
totalBytesRead,
@@ -535,7 +537,7 @@ public class ITestAzureBlobFileSystemRandomRead extends DependencyInjectedTest {
character = (character == 'z') ? 'a' : (char) ((int) character + 1);
}
- System.out.println(String.format("Creating test file %s of size: %d ", TEST_FILE_PATH, TEST_FILE_SIZE));
+ LOG.info(String.format("Creating test file %s of size: %d ", TEST_FILE_PATH, TEST_FILE_SIZE));
ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer();
try (FSDataOutputStream outputStream = fs.create(TEST_FILE_PATH)) {
@@ -544,7 +546,7 @@ public class ITestAzureBlobFileSystemRandomRead extends DependencyInjectedTest {
outputStream.write(buffer);
bytesWritten += buffer.length;
}
- System.out.println(String.format("Closing stream %s", outputStream));
+ LOG.info("Closing stream {}", outputStream);
ContractTestUtils.NanoTimer closeTimer
= new ContractTestUtils.NanoTimer();
outputStream.close();
@@ -578,4 +580,4 @@ public class ITestAzureBlobFileSystemRandomRead extends DependencyInjectedTest {
assertArrayEquals("Mismatch in read data", bufferV1, bufferV2);
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/873b519a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRename.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRename.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRename.java
index a0e648c..1a0edaf 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRename.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRename.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.fs.azurebfs;
-import java.io.FileNotFoundException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Callable;
@@ -26,93 +25,74 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
-import org.junit.Assert;
import org.junit.Test;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.assertIsDirectory;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.assertMkdirs;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.assertPathDoesNotExist;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.assertRenameOutcome;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.assertIsFile;
/**
* Test rename operation.
*/
-public class ITestAzureBlobFileSystemRename extends DependencyInjectedTest {
+public class ITestAzureBlobFileSystemRename extends
+ AbstractAbfsIntegrationTest {
public ITestAzureBlobFileSystemRename() {
- super();
}
- @Test(expected = FileNotFoundException.class)
+ @Test
public void testEnsureFileIsRenamed() throws Exception {
- final AzureBlobFileSystem fs = this.getFileSystem();
- fs.create(new Path("testfile"));
- fs.rename(new Path("testfile"), new Path("testfile2"));
-
- FileStatus fileStatus = fs.getFileStatus(new Path("testfile2"));
- assertNotNull(fileStatus);
-
- fs.getFileStatus(new Path("testfile"));
+ final AzureBlobFileSystem fs = getFileSystem();
+ Path src = path("testEnsureFileIsRenamed-src");
+ touch(src);
+ Path dest = path("testEnsureFileIsRenamed-dest");
+ fs.delete(dest, true);
+ assertRenameOutcome(fs, src, dest, true);
+
+ assertIsFile(fs, dest);
+ assertPathDoesNotExist(fs, "expected renamed", src);
}
@Test
- public void testRenameFile() throws Exception {
- final AzureBlobFileSystem fs = this.getFileSystem();
- fs.mkdirs(new Path("/testSrc"));
- fs.create(new Path("/testSrc/file1"));
-
- fs.rename(new Path("/testSrc"), new Path("/testDst"));
- FileStatus[] fileStatus = fs.listStatus(new Path("/testDst"));
- assertNotNull(fileStatus);
+ public void testRenameFileUnderDir() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ Path sourceDir = new Path("/testSrc");
+ assertMkdirs(fs, sourceDir);
+ String filename = "file1";
+ Path file1 = new Path(sourceDir, filename);
+ touch(file1);
+
+ Path destDir = new Path("/testDst");
+ assertRenameOutcome(fs, sourceDir, destDir, true);
+ FileStatus[] fileStatus = fs.listStatus(destDir);
+ assertNotNull("Null file status", fileStatus);
+ FileStatus status = fileStatus[0];
+ assertEquals("Wrong filename in " + status,
+ filename, status.getPath().getName());
}
@Test
- public void testRenameFileUsingUnicode() throws Exception {
- final AzureBlobFileSystem fs = this.getFileSystem();
- //known issue: ListStatus operation to folders/files whose name contains '?' will fail
- //This is because Auto rest client didn't encode '?' in the uri query parameters
- String[] folders1 = new String[]{"/%2c%26", "/ÖáΠ⇒", "/A +B", "/A~`!@#$%^&*()-_+={};:'>,<B"};
- String[] folders2 = new String[]{"/abcÖ⇒123", "/abcÖáΠ⇒123", "/B+ C", "/B~`!@#$%^&*()-_+={};:'>,<C"};
- String[] files = new String[]{"/%2c%27", "/中文", "/C +D", "/C~`!@#$%^&*()-_+={};:'>,<D"};
-
- for (int i = 0; i < 4; i++) {
- Path folderPath1 = new Path(folders1[i]);
- assertTrue(fs.mkdirs(folderPath1));
- assertTrue(fs.exists(folderPath1));
-
- Path filePath = new Path(folders1[i] + files[i]);
- fs.create(filePath);
- assertTrue(fs.exists(filePath));
-
- Path folderPath2 = new Path(folders2[i]);
- fs.rename(folderPath1, folderPath2);
- assertFalse(fs.exists(folderPath1));
- assertTrue(fs.exists(folderPath2));
-
- FileStatus[] fileStatus = fs.listStatus(folderPath2);
- assertEquals("/" + fileStatus[0].getPath().getName(), files[i]);
- assertNotNull(fileStatus);
- }
- }
-
- @Test(expected = FileNotFoundException.class)
public void testRenameDirectory() throws Exception {
- final AzureBlobFileSystem fs = this.getFileSystem();
+ final AzureBlobFileSystem fs = getFileSystem();
fs.mkdirs(new Path("testDir"));
- fs.mkdirs(new Path("testDir/test1"));
+ Path test1 = new Path("testDir/test1");
+ fs.mkdirs(test1);
fs.mkdirs(new Path("testDir/test1/test2"));
fs.mkdirs(new Path("testDir/test1/test2/test3"));
- Assert.assertTrue(fs.rename(new Path("testDir/test1"), new Path("testDir/test10")));
- fs.getFileStatus(new Path("testDir/test1"));
+ assertRenameOutcome(fs, test1,
+ new Path("testDir/test10"), true);
+ assertPathDoesNotExist(fs, "rename source dir", test1 );
}
- @Test(expected = FileNotFoundException.class)
+ @Test
public void testRenameFirstLevelDirectory() throws Exception {
- final AzureBlobFileSystem fs = this.getFileSystem();
- final List<Future> tasks = new ArrayList<>();
+ final AzureBlobFileSystem fs = getFileSystem();
+ final List<Future<Void>> tasks = new ArrayList<>();
ExecutorService es = Executors.newFixedThreadPool(10);
for (int i = 0; i < 1000; i++) {
@@ -120,7 +100,7 @@ public class ITestAzureBlobFileSystemRename extends DependencyInjectedTest {
Callable<Void> callable = new Callable<Void>() {
@Override
public Void call() throws Exception {
- fs.create(fileName);
+ touch(fileName);
return null;
}
};
@@ -133,20 +113,25 @@ public class ITestAzureBlobFileSystemRename extends DependencyInjectedTest {
}
es.shutdownNow();
- fs.rename(new Path("/test"), new Path("/renamedDir"));
+ Path source = new Path("/test");
+ Path dest = new Path("/renamedDir");
+ assertRenameOutcome(fs, source, dest, true);
- FileStatus[] files = fs.listStatus(new Path("/renamedDir"));
- Assert.assertEquals(files.length, 1000);
- fs.getFileStatus(new Path("/test"));
+ FileStatus[] files = fs.listStatus(dest);
+ assertEquals("Wrong number of files in listing", 1000, files.length);
+ assertPathDoesNotExist(fs, "rename source dir", source);
}
@Test
public void testRenameRoot() throws Exception {
- final AzureBlobFileSystem fs = this.getFileSystem();
- boolean renamed = fs.rename(new Path("/"), new Path("/ddd"));
- assertFalse(renamed);
-
- renamed = fs.rename(new Path(fs.getUri().toString() + "/"), new Path(fs.getUri().toString() + "/s"));
- assertFalse(renamed);
+ final AzureBlobFileSystem fs = getFileSystem();
+ assertRenameOutcome(fs,
+ new Path("/"),
+ new Path("/testRenameRoot"),
+ false);
+ assertRenameOutcome(fs,
+ new Path(fs.getUri().toString() + "/"),
+ new Path(fs.getUri().toString() + "/s"),
+ false);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/873b519a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRenameUnicode.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRenameUnicode.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRenameUnicode.java
new file mode 100644
index 0000000..0ac7fcf
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRenameUnicode.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs;
+
+import java.util.Arrays;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+
+import static org.apache.hadoop.fs.contract.ContractTestUtils.assertIsDirectory;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.assertIsFile;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.assertMkdirs;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.assertPathDoesNotExist;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.assertPathExists;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.assertRenameOutcome;
+
+/**
+ * Parameterized test of rename operations of unicode paths.
+ */
+@RunWith(Parameterized.class)
+public class ITestAzureBlobFileSystemRenameUnicode extends
+ AbstractAbfsIntegrationTest {
+
+ @Parameterized.Parameter
+ public String srcDir;
+
+ @Parameterized.Parameter(1)
+ public String destDir;
+
+ @Parameterized.Parameter(2)
+ public String filename;
+
+ @Parameterized.Parameters
+ public static Iterable<Object[]> params() {
+ return Arrays.asList(
+ new Object[][]{
+ {"/src", "/dest", "filename"},
+ {"/%2c%26", "/abcÖ⇒123", "%2c%27"},
+ {"/ÖáΠ⇒", "/abcÖáΠ⇒123", "中文"},
+ {"/A +B", "/B+ C", "C +D"},
+ {
+ "/A~`!@#$%^&*()-_+={};:'>,<B",
+ "/B~`!@#$%^&*()-_+={};:'>,<C",
+ "C~`!@#$%^&*()-_+={};:'>,<D"
+ }
+ });
+ }
+
+ public ITestAzureBlobFileSystemRenameUnicode() {
+ }
+
+ /**
+ * Known issue: ListStatus operation to folders/files whose name contains '?' will fail.
+ * This is because Auto rest client didn't encode '?' in the uri query parameters
+ */
+ @Test
+ public void testRenameFileUsingUnicode() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ Path folderPath1 = new Path(srcDir);
+ assertMkdirs(fs, folderPath1);
+ assertIsDirectory(fs, folderPath1);
+ Path filePath = new Path(folderPath1 + "/" + filename);
+ touch(filePath);
+ assertIsFile(fs, filePath);
+
+ Path folderPath2 = new Path(destDir);
+ assertRenameOutcome(fs, folderPath1, folderPath2, true);
+ assertPathDoesNotExist(fs, "renamed", folderPath1);
+ assertIsDirectory(fs, folderPath2);
+ assertPathExists(fs, "renamed file", new Path(folderPath2 + "/" + filename));
+
+ FileStatus[] fileStatus = fs.listStatus(folderPath2);
+ assertNotNull(fileStatus);
+ assertTrue("Empty listing returned from listStatus(\"" + folderPath2 + "\")",
+ fileStatus.length > 0);
+ assertEquals(fileStatus[0].getPath().getName(), filename);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/873b519a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemInitialization.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemInitialization.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemInitialization.java
index 29af1b8..3a44909 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemInitialization.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemInitialization.java
@@ -20,9 +20,9 @@ package org.apache.hadoop.fs.azurebfs;
import java.net.URI;
-import org.junit.Assert;
import org.junit.Test;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes;
@@ -30,30 +30,45 @@ import org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes;
/**
* Test AzureBlobFileSystem initialization.
*/
-public class ITestFileSystemInitialization extends DependencyInjectedTest {
+public class ITestFileSystemInitialization extends AbstractAbfsIntegrationTest {
public ITestFileSystemInitialization() {
super();
}
@Test
public void ensureAzureBlobFileSystemIsInitialized() throws Exception {
- final FileSystem fs = this.getFileSystem();
- final String accountName = this.getAccountName();
- final String filesystem = this.getFileSystemName();
+ final AzureBlobFileSystem fs = getFileSystem();
+ final String accountName = getAccountName();
+ final String filesystem = getFileSystemName();
- Assert.assertEquals(fs.getUri(), new URI(FileSystemUriSchemes.ABFS_SCHEME, filesystem + "@" + accountName, null, null, null));
- Assert.assertNotNull(fs.getWorkingDirectory());
+ assertEquals(fs.getUri(),
+ new URI(FileSystemUriSchemes.ABFS_SCHEME,
+ filesystem + "@" + accountName,
+ null,
+ null,
+ null));
+ assertNotNull("working directory", fs.getWorkingDirectory());
}
@Test
public void ensureSecureAzureBlobFileSystemIsInitialized() throws Exception {
- final String accountName = this.getAccountName();
- final String filesystem = this.getFileSystemName();
- final URI defaultUri = new URI(FileSystemUriSchemes.ABFS_SECURE_SCHEME, filesystem + "@" + accountName, null, null, null);
- this.getConfiguration().set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri.toString());
-
- final FileSystem fs = this.getFileSystem();
- Assert.assertEquals(fs.getUri(), new URI(FileSystemUriSchemes.ABFS_SECURE_SCHEME, filesystem + "@" + accountName, null, null, null));
- Assert.assertNotNull(fs.getWorkingDirectory());
+ final String accountName = getAccountName();
+ final String filesystem = getFileSystemName();
+ final URI defaultUri = new URI(FileSystemUriSchemes.ABFS_SECURE_SCHEME,
+ filesystem + "@" + accountName,
+ null,
+ null,
+ null);
+ Configuration conf = getConfiguration();
+ conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri.toString());
+
+ try(SecureAzureBlobFileSystem fs = (SecureAzureBlobFileSystem) FileSystem.newInstance(conf)) {
+ assertEquals(fs.getUri(), new URI(FileSystemUriSchemes.ABFS_SECURE_SCHEME,
+ filesystem + "@" + accountName,
+ null,
+ null,
+ null));
+ assertNotNull("working directory", fs.getWorkingDirectory());
+ }
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/873b519a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemProperties.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemProperties.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemProperties.java
index 62d967e..1c71125 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemProperties.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemProperties.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.fs.azurebfs;
import java.util.Hashtable;
-import org.junit.Assert;
import org.junit.Ignore;
import org.junit.Test;
@@ -34,31 +33,29 @@ import static org.junit.Assert.assertEquals;
/**
* Test FileSystemProperties.
*/
-public class ITestFileSystemProperties extends DependencyInjectedTest {
+public class ITestFileSystemProperties extends AbstractAbfsIntegrationTest {
private static final int TEST_DATA = 100;
private static final Path TEST_PATH = new Path("/testfile");
public ITestFileSystemProperties() {
- super();
}
@Test
public void testReadWriteBytesToFileAndEnsureThreadPoolCleanup() throws Exception {
- final AzureBlobFileSystem fs = this.getFileSystem();
+ final AzureBlobFileSystem fs = getFileSystem();
testWriteOneByteToFileAndEnsureThreadPoolCleanup();
- FSDataInputStream inputStream = fs.open(TEST_PATH, 4 * 1024 * 1024);
- int i = inputStream.read();
-
- assertEquals(TEST_DATA, i);
+ try(FSDataInputStream inputStream = fs.open(TEST_PATH, 4 * 1024 * 1024)) {
+ int i = inputStream.read();
+ assertEquals(TEST_DATA, i);
+ }
}
@Test
public void testWriteOneByteToFileAndEnsureThreadPoolCleanup() throws Exception {
- final AzureBlobFileSystem fs = this.getFileSystem();
- FSDataOutputStream stream = fs.create(TEST_PATH);
-
- stream.write(TEST_DATA);
- stream.close();
+ final AzureBlobFileSystem fs = getFileSystem();
+ try(FSDataOutputStream stream = fs.create(TEST_PATH)) {
+ stream.write(TEST_DATA);
+ }
FileStatus fileStatus = fs.getFileStatus(TEST_PATH);
assertEquals(1, fileStatus.getLen());
@@ -67,60 +64,60 @@ public class ITestFileSystemProperties extends DependencyInjectedTest {
@Test
@Ignore("JDK7 doesn't support PATCH, so PUT is used. Fix is applied in latest test tenant")
public void testBase64FileSystemProperties() throws Exception {
- final AzureBlobFileSystem fs = this.getFileSystem();
+ final AzureBlobFileSystem fs = getFileSystem();
final Hashtable<String, String> properties = new Hashtable<>();
properties.put("key", "{ value: value }");
fs.getAbfsStore().setFilesystemProperties(properties);
Hashtable<String, String> fetchedProperties = fs.getAbfsStore().getFilesystemProperties();
- Assert.assertEquals(properties, fetchedProperties);
+ assertEquals(properties, fetchedProperties);
}
@Test
public void testBase64PathProperties() throws Exception {
- final AzureBlobFileSystem fs = this.getFileSystem();
+ final AzureBlobFileSystem fs = getFileSystem();
final Hashtable<String, String> properties = new Hashtable<>();
properties.put("key", "{ value: valueTest }");
- fs.create(TEST_PATH);
+ touch(TEST_PATH);
fs.getAbfsStore().setPathProperties(TEST_PATH, properties);
Hashtable<String, String> fetchedProperties =
fs.getAbfsStore().getPathProperties(TEST_PATH);
- Assert.assertEquals(properties, fetchedProperties);
+ assertEquals(properties, fetchedProperties);
}
@Test (expected = Exception.class)
public void testBase64InvalidFileSystemProperties() throws Exception {
- final AzureBlobFileSystem fs = this.getFileSystem();
+ final AzureBlobFileSystem fs = getFileSystem();
final Hashtable<String, String> properties = new Hashtable<>();
properties.put("key", "{ value: value歲 }");
fs.getAbfsStore().setFilesystemProperties(properties);
Hashtable<String, String> fetchedProperties = fs.getAbfsStore().getFilesystemProperties();
- Assert.assertEquals(properties, fetchedProperties);
+ assertEquals(properties, fetchedProperties);
}
@Test (expected = Exception.class)
public void testBase64InvalidPathProperties() throws Exception {
- final AzureBlobFileSystem fs = this.getFileSystem();
+ final AzureBlobFileSystem fs = getFileSystem();
final Hashtable<String, String> properties = new Hashtable<>();
properties.put("key", "{ value: valueTest兩 }");
- fs.create(TEST_PATH);
+ touch(TEST_PATH);
fs.getAbfsStore().setPathProperties(TEST_PATH, properties);
Hashtable<String, String> fetchedProperties = fs.getAbfsStore().getPathProperties(TEST_PATH);
- Assert.assertEquals(properties, fetchedProperties);
+ assertEquals(properties, fetchedProperties);
}
@Test
public void testSetFileSystemProperties() throws Exception {
- final AzureBlobFileSystem fs = this.getFileSystem();
+ final AzureBlobFileSystem fs = getFileSystem();
final Hashtable<String, String> properties = new Hashtable<>();
properties.put("containerForDevTest", "true");
fs.getAbfsStore().setFilesystemProperties(properties);
Hashtable<String, String> fetchedProperties = fs.getAbfsStore().getFilesystemProperties();
- Assert.assertEquals(properties, fetchedProperties);
+ assertEquals(properties, fetchedProperties);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/873b519a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemRegistration.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemRegistration.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemRegistration.java
index ef61e52..5d1cf91 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemRegistration.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemRegistration.java
@@ -20,10 +20,9 @@ package org.apache.hadoop.fs.azurebfs;
import java.net.URI;
-import org.junit.Assert;
import org.junit.Test;
-import org.apache.hadoop.fs.AbstractFileSystem;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileSystem;
@@ -31,33 +30,76 @@ import org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes;
/**
* Test AzureBlobFileSystem registration.
+ * Use casts to have interesting stack traces on failures.
*/
-public class ITestFileSystemRegistration extends DependencyInjectedTest {
+public class ITestFileSystemRegistration extends AbstractAbfsIntegrationTest {
+
+ protected static final String ABFS = "org.apache.hadoop.fs.azurebfs.Abfs";
+ protected static final String ABFSS = "org.apache.hadoop.fs.azurebfs.Abfss";
+
public ITestFileSystemRegistration() throws Exception {
- super();
+ }
+
+ private void assertConfigMatches(Configuration conf, String key, String expected) {
+ String v = conf.get(key);
+ assertNotNull("No value for key " + key, v);
+ assertEquals("Wrong value for key " + key, expected, v);
+ }
+
+ @Test
+ public void testAbfsFileSystemRegistered() throws Throwable {
+ assertConfigMatches(new Configuration(true),
+ "fs.abfs.impl",
+ "org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem");
+ }
+
+ @Test
+ public void testSecureAbfsFileSystemRegistered() throws Throwable {
+ assertConfigMatches(new Configuration(true),
+ "fs.abfss.impl",
+ "org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem");
+ }
+
+ @Test
+ public void testAbfsFileContextRegistered() throws Throwable {
+ assertConfigMatches(new Configuration(true),
+ "fs.AbstractFileSystem.abfs.impl",
+ ABFS);
+ }
+
+ @Test
+ public void testSecureAbfsFileContextRegistered() throws Throwable {
+ assertConfigMatches(new Configuration(true),
+ "fs.AbstractFileSystem.abfss.impl",
+ ABFSS);
}
@Test
public void ensureAzureBlobFileSystemIsDefaultFileSystem() throws Exception {
- FileSystem fs = FileSystem.get(this.getConfiguration());
- Assert.assertTrue(fs instanceof AzureBlobFileSystem);
+ AzureBlobFileSystem fs = (AzureBlobFileSystem) FileSystem.get(getConfiguration());
+ assertNotNull("filesystem", fs);
- AbstractFileSystem afs = FileContext.getFileContext(this.getConfiguration()).getDefaultFileSystem();
- Assert.assertTrue(afs instanceof Abfs);
+ Abfs afs = (Abfs) FileContext.getFileContext(getConfiguration()).getDefaultFileSystem();
+ assertNotNull("filecontext", afs);
}
@Test
public void ensureSecureAzureBlobFileSystemIsDefaultFileSystem() throws Exception {
- final String accountName = this.getAccountName();
- final String fileSystemName = this.getFileSystemName();
-
- final URI defaultUri = new URI(FileSystemUriSchemes.ABFS_SECURE_SCHEME, fileSystemName + "@" + accountName, null, null, null);
- this.getConfiguration().set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri.toString());
+ final String accountName = getAccountName();
+ final String fileSystemName = getFileSystemName();
- FileSystem fs = FileSystem.get(this.getConfiguration());
- Assert.assertTrue(fs instanceof SecureAzureBlobFileSystem);
+ final URI defaultUri = new URI(FileSystemUriSchemes.ABFS_SECURE_SCHEME,
+ fileSystemName + "@" + accountName,
+ null,
+ null,
+ null);
+ getConfiguration().set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY,
+ defaultUri.toString());
- AbstractFileSystem afs = FileContext.getFileContext(this.getConfiguration()).getDefaultFileSystem();
- Assert.assertTrue(afs instanceof Abfss);
+ SecureAzureBlobFileSystem fs = (SecureAzureBlobFileSystem) FileSystem.get(
+ getConfiguration());
+ assertNotNull("filesystem", fs);
+ Abfss afs = (Abfss) FileContext.getFileContext(getConfiguration()).getDefaultFileSystem();
+ assertNotNull("filecontext", afs);
}
-}
\ No newline at end of file
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org