You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by tm...@apache.org on 2018/08/11 05:37:45 UTC
[41/50] [abbrv] hadoop git commit: HADOOP-15407. HADOOP-15540.
Support Windows Azure Storage - Blob file system "ABFS" in Hadoop: Core
Commit.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdf5f4c3/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemListStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemListStatus.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemListStatus.java
new file mode 100644
index 0000000..6059766
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemListStatus.java
@@ -0,0 +1,132 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs;
+
+import java.io.FileNotFoundException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
+
+/**
+ * Test listStatus operation.
+ */
+public class ITestAzureBlobFileSystemListStatus extends DependencyInjectedTest {
+ private static final int TEST_FILES_NUMBER = 6000;
+ public ITestAzureBlobFileSystemListStatus() {
+ super();
+ }
+
+ @Test
+ public void testListPath() throws Exception {
+ final AzureBlobFileSystem fs = this.getFileSystem();
+ final List<Future> tasks = new ArrayList<>();
+
+ ExecutorService es = Executors.newFixedThreadPool(10);
+ for (int i = 0; i < TEST_FILES_NUMBER; i++) {
+ final Path fileName = new Path("/test" + i);
+ Callable<Void> callable = new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ fs.create(fileName);
+ return null;
+ }
+ };
+
+ tasks.add(es.submit(callable));
+ }
+
+ for (Future<Void> task : tasks) {
+ task.get();
+ }
+
+ es.shutdownNow();
+ FileStatus[] files = fs.listStatus(new Path("/"));
+ Assert.assertEquals(files.length, TEST_FILES_NUMBER + 1 /* user directory */);
+ }
+
+ @Test
+ public void testListFileVsListDir() throws Exception {
+ final AzureBlobFileSystem fs = this.getFileSystem();
+ fs.create(new Path("/testFile"));
+
+ FileStatus[] testFiles = fs.listStatus(new Path("/testFile"));
+ Assert.assertEquals(testFiles.length, 1);
+ Assert.assertFalse(testFiles[0].isDirectory());
+ }
+
+ @Test
+ public void testListFileVsListDir2() throws Exception {
+ final AzureBlobFileSystem fs = this.getFileSystem();
+ fs.mkdirs(new Path("/testFolder"));
+ fs.mkdirs(new Path("/testFolder/testFolder2"));
+ fs.mkdirs(new Path("/testFolder/testFolder2/testFolder3"));
+ fs.create(new Path("/testFolder/testFolder2/testFolder3/testFile"));
+
+ FileStatus[] testFiles = fs.listStatus(new Path("/testFolder/testFolder2/testFolder3/testFile"));
+ Assert.assertEquals(testFiles.length, 1);
+ Assert.assertEquals(testFiles[0].getPath(), new Path(this.getTestUrl(),
+ "/testFolder/testFolder2/testFolder3/testFile"));
+ Assert.assertFalse(testFiles[0].isDirectory());
+ }
+
+ @Test(expected = FileNotFoundException.class)
+ public void testListNonExistentDir() throws Exception {
+ final AzureBlobFileSystem fs = this.getFileSystem();
+ fs.listStatus(new Path("/testFile/"));
+ }
+
+ @Test
+ public void testListFiles() throws Exception {
+ final AzureBlobFileSystem fs = this.getFileSystem();
+ fs.mkdirs(new Path("/test"));
+
+ FileStatus[] fileStatuses = fs.listStatus(new Path("/"));
+ assertEquals(fileStatuses.length, 2);
+
+ fs.mkdirs(new Path("/test/sub"));
+ fileStatuses = fs.listStatus(new Path("/test"));
+ assertEquals(fileStatuses.length, 1);
+ assertEquals(fileStatuses[0].getPath().getName(), "sub");
+ assertTrue(fileStatuses[0].isDirectory());
+ assertEquals(fileStatuses[0].getLen(), 0);
+
+ fs.create(new Path("/test/f"));
+ fileStatuses = fs.listStatus(new Path("/test"));
+ assertEquals(fileStatuses.length, 2);
+ assertEquals(fileStatuses[0].getPath().getName(), "f");
+ assertFalse(fileStatuses[0].isDirectory());
+ assertEquals(fileStatuses[0].getLen(), 0);
+ assertEquals(fileStatuses[1].getPath().getName(), "sub");
+ assertTrue(fileStatuses[1].isDirectory());
+ assertEquals(fileStatuses[1].getLen(), 0);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdf5f4c3/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemMkDir.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemMkDir.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemMkDir.java
new file mode 100644
index 0000000..b61908c
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemMkDir.java
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs;
+
+import java.util.concurrent.Callable;
+
+import org.junit.Test;
+
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.Path;
+
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Test mkdir operation.
+ */
+public class ITestAzureBlobFileSystemMkDir extends DependencyInjectedTest {
+ public ITestAzureBlobFileSystemMkDir() {
+ super();
+ }
+
+ @Test
+ public void testCreateDirWithExistingDir() throws Exception {
+ final AzureBlobFileSystem fs = this.getFileSystem();
+ assertTrue(fs.mkdirs(new Path("testFolder")));
+ assertTrue(fs.mkdirs(new Path("testFolder")));
+ }
+
+ @Test(expected = FileAlreadyExistsException.class)
+ public void createDirectoryUnderFile() throws Exception {
+ final AzureBlobFileSystem fs = this.getFileSystem();
+ fs.create(new Path("testFile"));
+ fs.mkdirs(new Path("testFile/TestDirectory"));
+ }
+
+ @Test
+ public void testCreateDirectoryOverExistingFiles() throws Exception {
+ final AzureBlobFileSystem fs = this.getFileSystem();
+ fs.create(new Path("/testPath"));
+ FileAlreadyExistsException ex = intercept(
+ FileAlreadyExistsException.class,
+ new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ fs.mkdirs(new Path("/testPath"));
+ return null;
+ }
+ });
+
+ assertTrue(ex instanceof FileAlreadyExistsException);
+
+ fs.create(new Path("/testPath1/file1"));
+ ex = intercept(
+ FileAlreadyExistsException.class,
+ new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ fs.mkdirs(new Path("/testPath1/file1"));
+ return null;
+ }
+ });
+
+ assertTrue(ex instanceof FileAlreadyExistsException);
+ }
+
+ @Test
+ public void testCreateRoot() throws Exception {
+ final AzureBlobFileSystem fs = this.getFileSystem();
+ assertTrue(fs.mkdirs(new Path("/")));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdf5f4c3/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemOpen.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemOpen.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemOpen.java
new file mode 100644
index 0000000..fef7f47
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemOpen.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs;
+
+import java.io.FileNotFoundException;
+
+import org.junit.Test;
+
+import org.apache.hadoop.fs.Path;
+
+/**
+ * Test open operation.
+ */
+public class ITestAzureBlobFileSystemOpen extends DependencyInjectedTest {
+ public ITestAzureBlobFileSystemOpen() throws Exception {
+ super();
+ }
+
+ @Test(expected = FileNotFoundException.class)
+ public void testOpenDirectory() throws Exception {
+ final AzureBlobFileSystem fs = this.getFileSystem();
+ fs.mkdirs(new Path("testFolder"));
+ fs.open(new Path("testFolder"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdf5f4c3/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRandomRead.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRandomRead.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRandomRead.java
new file mode 100644
index 0000000..9477587
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRandomRead.java
@@ -0,0 +1,582 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.azurebfs;
+
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FSExceptionMessages;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.azure.NativeAzureFileSystem;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.junit.Test;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.util.Random;
+import java.util.concurrent.Callable;
+
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertArrayEquals;
+
+
+/**
+ * Test random read operation.
+ */
+public class ITestAzureBlobFileSystemRandomRead extends DependencyInjectedTest {
+ private static final int KILOBYTE = 1024;
+ private static final int MEGABYTE = KILOBYTE * KILOBYTE;
+ private static final long TEST_FILE_SIZE = 8 * MEGABYTE;
+ private static final int MAX_ELAPSEDTIMEMS = 20;
+ private static final int SEQUENTIAL_READ_BUFFER_SIZE = 16 * KILOBYTE;
+ private static final int CREATE_BUFFER_SIZE = 26 * KILOBYTE;
+
+ private static final int SEEK_POSITION_ONE = 2* KILOBYTE;
+ private static final int SEEK_POSITION_TWO = 5 * KILOBYTE;
+ private static final int SEEK_POSITION_THREE = 10 * KILOBYTE;
+ private static final int SEEK_POSITION_FOUR = 4100 * KILOBYTE;
+
+ private static final Path TEST_FILE_PATH = new Path(
+ "/TestRandomRead.txt");
+ private static final String WASB = "WASB";
+ private static final String ABFS = "ABFS";
+ private static long testFileLength = 0;
+
+ public ITestAzureBlobFileSystemRandomRead() throws Exception {
+ super();
+ }
+
+ @Test
+ public void testBasicRead() throws Exception {
+ assumeHugeFileExists();
+
+ try (FSDataInputStream inputStream = this.getFileSystem().open(TEST_FILE_PATH)) {
+ byte[] buffer = new byte[3 * MEGABYTE];
+
+ // forward seek and read a kilobyte into first kilobyte of bufferV2
+ inputStream.seek(5 * MEGABYTE);
+ int numBytesRead = inputStream.read(buffer, 0, KILOBYTE);
+ assertEquals(KILOBYTE, numBytesRead);
+
+ int len = MEGABYTE;
+ int offset = buffer.length - len;
+
+ // reverse seek and read a megabyte into last megabyte of bufferV1
+ inputStream.seek(3 * MEGABYTE);
+ numBytesRead = inputStream.read(buffer, offset, len);
+ assertEquals(len, numBytesRead);
+ }
+ }
+
+ /**
+ * Validates the implementation of random read in ABFS
+ * @throws IOException
+ */
+ @Test
+ public void testRandomRead() throws Exception {
+ assumeHugeFileExists();
+ try (
+ FSDataInputStream inputStreamV1
+ = this.getFileSystem().open(TEST_FILE_PATH);
+ FSDataInputStream inputStreamV2
+ = this.getWasbFileSystem().open(TEST_FILE_PATH);
+ ) {
+ final int bufferSize = 4 * KILOBYTE;
+ byte[] bufferV1 = new byte[bufferSize];
+ byte[] bufferV2 = new byte[bufferV1.length];
+
+ verifyConsistentReads(inputStreamV1, inputStreamV2, bufferV1, bufferV2);
+
+ inputStreamV1.seek(0);
+ inputStreamV2.seek(0);
+
+ verifyConsistentReads(inputStreamV1, inputStreamV2, bufferV1, bufferV2);
+
+ verifyConsistentReads(inputStreamV1, inputStreamV2, bufferV1, bufferV2);
+
+ inputStreamV1.seek(SEEK_POSITION_ONE);
+ inputStreamV2.seek(SEEK_POSITION_ONE);
+
+ inputStreamV1.seek(0);
+ inputStreamV2.seek(0);
+
+ verifyConsistentReads(inputStreamV1, inputStreamV2, bufferV1, bufferV2);
+
+ verifyConsistentReads(inputStreamV1, inputStreamV2, bufferV1, bufferV2);
+
+ verifyConsistentReads(inputStreamV1, inputStreamV2, bufferV1, bufferV2);
+
+ inputStreamV1.seek(SEEK_POSITION_TWO);
+ inputStreamV2.seek(SEEK_POSITION_TWO);
+
+ verifyConsistentReads(inputStreamV1, inputStreamV2, bufferV1, bufferV2);
+
+ inputStreamV1.seek(SEEK_POSITION_THREE);
+ inputStreamV2.seek(SEEK_POSITION_THREE);
+
+ verifyConsistentReads(inputStreamV1, inputStreamV2, bufferV1, bufferV2);
+
+ verifyConsistentReads(inputStreamV1, inputStreamV2, bufferV1, bufferV2);
+
+ inputStreamV1.seek(SEEK_POSITION_FOUR);
+ inputStreamV2.seek(SEEK_POSITION_FOUR);
+
+ verifyConsistentReads(inputStreamV1, inputStreamV2, bufferV1, bufferV2);
+ }
+ }
+
+ /**
+ * Validates the implementation of Seekable.seekToNewSource
+ * @throws IOException
+ */
+ @Test
+ public void testSeekToNewSource() throws Exception {
+ assumeHugeFileExists();
+ try (FSDataInputStream inputStream = this.getFileSystem().open(TEST_FILE_PATH)) {
+ assertFalse(inputStream.seekToNewSource(0));
+ }
+ }
+
+ /**
+ * Validates the implementation of InputStream.skip and ensures there is no
+ * network I/O for AbfsInputStream
+ * @throws Exception
+ */
+ @Test
+ public void testSkipBounds() throws Exception {
+ assumeHugeFileExists();
+ try (FSDataInputStream inputStream = this.getFileSystem().open(TEST_FILE_PATH)) {
+ ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer();
+
+ long skipped = inputStream.skip(-1);
+ assertEquals(0, skipped);
+
+ skipped = inputStream.skip(0);
+ assertEquals(0, skipped);
+
+ assertTrue(testFileLength > 0);
+
+ skipped = inputStream.skip(testFileLength);
+ assertEquals(testFileLength, skipped);
+
+ intercept(EOFException.class,
+ new Callable<Long>() {
+ @Override
+ public Long call() throws Exception {
+ return inputStream.skip(1);
+ }
+ }
+ );
+ long elapsedTimeMs = timer.elapsedTimeMs();
+ assertTrue(
+ String.format(
+ "There should not be any network I/O (elapsedTimeMs=%1$d).",
+ elapsedTimeMs),
+ elapsedTimeMs < MAX_ELAPSEDTIMEMS);
+ }
+ }
+
+ /**
+ * Validates the implementation of Seekable.seek and ensures there is no
+ * network I/O for forward seek.
+ * @throws Exception
+ */
+ @Test
+ public void testValidateSeekBounds() throws Exception {
+ assumeHugeFileExists();
+ try (FSDataInputStream inputStream = this.getFileSystem().open(TEST_FILE_PATH)) {
+ ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer();
+
+ inputStream.seek(0);
+ assertEquals(0, inputStream.getPos());
+
+ intercept(EOFException.class,
+ FSExceptionMessages.NEGATIVE_SEEK,
+ new Callable<FSDataInputStream>() {
+ @Override
+ public FSDataInputStream call() throws Exception {
+ inputStream.seek(-1);
+ return inputStream;
+ }
+ }
+ );
+
+ assertTrue("Test file length only " + testFileLength, testFileLength > 0);
+ inputStream.seek(testFileLength);
+ assertEquals(testFileLength, inputStream.getPos());
+
+ intercept(EOFException.class,
+ FSExceptionMessages.CANNOT_SEEK_PAST_EOF,
+ new Callable<FSDataInputStream>() {
+ @Override
+ public FSDataInputStream call() throws Exception {
+ inputStream.seek(testFileLength + 1);
+ return inputStream;
+ }
+ }
+ );
+
+ long elapsedTimeMs = timer.elapsedTimeMs();
+ assertTrue(
+ String.format(
+ "There should not be any network I/O (elapsedTimeMs=%1$d).",
+ elapsedTimeMs),
+ elapsedTimeMs < MAX_ELAPSEDTIMEMS);
+ }
+ }
+
+ /**
+ * Validates the implementation of Seekable.seek, Seekable.getPos,
+ * and InputStream.available.
+ * @throws Exception
+ */
+ @Test
+ public void testSeekAndAvailableAndPosition() throws Exception {
+ assumeHugeFileExists();
+ try (FSDataInputStream inputStream = this.getFileSystem().open(TEST_FILE_PATH)) {
+ byte[] expected1 = {(byte) 'a', (byte) 'b', (byte) 'c'};
+ byte[] expected2 = {(byte) 'd', (byte) 'e', (byte) 'f'};
+ byte[] expected3 = {(byte) 'b', (byte) 'c', (byte) 'd'};
+ byte[] expected4 = {(byte) 'g', (byte) 'h', (byte) 'i'};
+ byte[] buffer = new byte[3];
+
+ int bytesRead = inputStream.read(buffer);
+ assertEquals(buffer.length, bytesRead);
+ assertArrayEquals(expected1, buffer);
+ assertEquals(buffer.length, inputStream.getPos());
+ assertEquals(testFileLength - inputStream.getPos(),
+ inputStream.available());
+
+ bytesRead = inputStream.read(buffer);
+ assertEquals(buffer.length, bytesRead);
+ assertArrayEquals(expected2, buffer);
+ assertEquals(2 * buffer.length, inputStream.getPos());
+ assertEquals(testFileLength - inputStream.getPos(),
+ inputStream.available());
+
+ // reverse seek
+ int seekPos = 0;
+ inputStream.seek(seekPos);
+
+ bytesRead = inputStream.read(buffer);
+ assertEquals(buffer.length, bytesRead);
+ assertArrayEquals(expected1, buffer);
+ assertEquals(buffer.length + seekPos, inputStream.getPos());
+ assertEquals(testFileLength - inputStream.getPos(),
+ inputStream.available());
+
+ // reverse seek
+ seekPos = 1;
+ inputStream.seek(seekPos);
+
+ bytesRead = inputStream.read(buffer);
+ assertEquals(buffer.length, bytesRead);
+ assertArrayEquals(expected3, buffer);
+ assertEquals(buffer.length + seekPos, inputStream.getPos());
+ assertEquals(testFileLength - inputStream.getPos(),
+ inputStream.available());
+
+ // forward seek
+ seekPos = 6;
+ inputStream.seek(seekPos);
+
+ bytesRead = inputStream.read(buffer);
+ assertEquals(buffer.length, bytesRead);
+ assertArrayEquals(expected4, buffer);
+ assertEquals(buffer.length + seekPos, inputStream.getPos());
+ assertEquals(testFileLength - inputStream.getPos(),
+ inputStream.available());
+ }
+ }
+
+ /**
+ * Validates the implementation of InputStream.skip, Seekable.getPos,
+ * and InputStream.available.
+ * @throws IOException
+ */
+ @Test
+ public void testSkipAndAvailableAndPosition() throws Exception {
+ assumeHugeFileExists();
+ try (FSDataInputStream inputStream = this.getFileSystem().open(TEST_FILE_PATH)) {
+ byte[] expected1 = {(byte) 'a', (byte) 'b', (byte) 'c'};
+ byte[] expected2 = {(byte) 'd', (byte) 'e', (byte) 'f'};
+ byte[] expected3 = {(byte) 'b', (byte) 'c', (byte) 'd'};
+ byte[] expected4 = {(byte) 'g', (byte) 'h', (byte) 'i'};
+
+ assertEquals(testFileLength, inputStream.available());
+ assertEquals(0, inputStream.getPos());
+
+ int n = 3;
+ long skipped = inputStream.skip(n);
+
+ assertEquals(skipped, inputStream.getPos());
+ assertEquals(testFileLength - inputStream.getPos(),
+ inputStream.available());
+ assertEquals(skipped, n);
+
+ byte[] buffer = new byte[3];
+ int bytesRead = inputStream.read(buffer);
+ assertEquals(buffer.length, bytesRead);
+ assertArrayEquals(expected2, buffer);
+ assertEquals(buffer.length + skipped, inputStream.getPos());
+ assertEquals(testFileLength - inputStream.getPos(),
+ inputStream.available());
+
+ // does skip still work after seek?
+ int seekPos = 1;
+ inputStream.seek(seekPos);
+
+ bytesRead = inputStream.read(buffer);
+ assertEquals(buffer.length, bytesRead);
+ assertArrayEquals(expected3, buffer);
+ assertEquals(buffer.length + seekPos, inputStream.getPos());
+ assertEquals(testFileLength - inputStream.getPos(),
+ inputStream.available());
+
+ long currentPosition = inputStream.getPos();
+ n = 2;
+ skipped = inputStream.skip(n);
+
+ assertEquals(currentPosition + skipped, inputStream.getPos());
+ assertEquals(testFileLength - inputStream.getPos(),
+ inputStream.available());
+ assertEquals(skipped, n);
+
+ bytesRead = inputStream.read(buffer);
+ assertEquals(buffer.length, bytesRead);
+ assertArrayEquals(expected4, buffer);
+ assertEquals(buffer.length + skipped + currentPosition,
+ inputStream.getPos());
+ assertEquals(testFileLength - inputStream.getPos(),
+ inputStream.available());
+ }
+ }
+
+ /**
+ * Ensures parity in the performance of sequential read after reverse seek for
+ * abfs of the AbfsInputStream.
+ * @throws IOException
+ */
+ @Test
+ public void testSequentialReadAfterReverseSeekPerformance()
+ throws Exception {
+ assumeHugeFileExists();
+ final int maxAttempts = 10;
+ final double maxAcceptableRatio = 1.01;
+ double beforeSeekElapsedMs = 0, afterSeekElapsedMs = 0;
+ double ratio = Double.MAX_VALUE;
+ for (int i = 0; i < maxAttempts && ratio >= maxAcceptableRatio; i++) {
+ beforeSeekElapsedMs = sequentialRead(ABFS,
+ this.getFileSystem(), false);
+ afterSeekElapsedMs = sequentialRead(ABFS,
+ this.getFileSystem(), true);
+ ratio = afterSeekElapsedMs / beforeSeekElapsedMs;
+ System.out.println((String.format(
+ "beforeSeekElapsedMs=%1$d, afterSeekElapsedMs=%2$d, ratio=%3$.2f",
+ (long) beforeSeekElapsedMs,
+ (long) afterSeekElapsedMs,
+ ratio)));
+ }
+ assertTrue(String.format(
+ "Performance of ABFS stream after reverse seek is not acceptable:"
+ + " beforeSeekElapsedMs=%1$d, afterSeekElapsedMs=%2$d,"
+ + " ratio=%3$.2f",
+ (long) beforeSeekElapsedMs,
+ (long) afterSeekElapsedMs,
+ ratio),
+ ratio < maxAcceptableRatio);
+ }
+
+ @Test
+ public void testRandomReadPerformance() throws Exception {
+ createTestFile();
+ assumeHugeFileExists();
+
+ final AzureBlobFileSystem abFs = this.getFileSystem();
+ final NativeAzureFileSystem wasbFs = this.getWasbFileSystem();
+
+ final int maxAttempts = 10;
+ final double maxAcceptableRatio = 1.025;
+ double v1ElapsedMs = 0, v2ElapsedMs = 0;
+ double ratio = Double.MAX_VALUE;
+ for (int i = 0; i < maxAttempts && ratio >= maxAcceptableRatio; i++) {
+ v1ElapsedMs = randomRead(1, wasbFs);
+ v2ElapsedMs = randomRead(2, abFs);
+
+ ratio = v2ElapsedMs / v1ElapsedMs;
+
+ System.out.println(String.format(
+ "v1ElapsedMs=%1$d, v2ElapsedMs=%2$d, ratio=%3$.2f",
+ (long) v1ElapsedMs,
+ (long) v2ElapsedMs,
+ ratio));
+ }
+ assertTrue(String.format(
+ "Performance of version 2 is not acceptable: v1ElapsedMs=%1$d,"
+ + " v2ElapsedMs=%2$d, ratio=%3$.2f",
+ (long) v1ElapsedMs,
+ (long) v2ElapsedMs,
+ ratio),
+ ratio < maxAcceptableRatio);
+ }
+
+
+ private long sequentialRead(String version,
+ FileSystem fs,
+ boolean afterReverseSeek) throws IOException {
+ byte[] buffer = new byte[SEQUENTIAL_READ_BUFFER_SIZE];
+ long totalBytesRead = 0;
+ long bytesRead = 0;
+
+ try(FSDataInputStream inputStream = fs.open(TEST_FILE_PATH)) {
+ if (afterReverseSeek) {
+ while (bytesRead > 0 && totalBytesRead < 4 * MEGABYTE) {
+ bytesRead = inputStream.read(buffer);
+ totalBytesRead += bytesRead;
+ }
+ totalBytesRead = 0;
+ inputStream.seek(0);
+ }
+
+ ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer();
+ while ((bytesRead = inputStream.read(buffer)) > 0) {
+ totalBytesRead += bytesRead;
+ }
+ long elapsedTimeMs = timer.elapsedTimeMs();
+
+ System.out.println(String.format(
+ "v%1$s: bytesRead=%2$d, elapsedMs=%3$d, Mbps=%4$.2f,"
+ + " afterReverseSeek=%5$s",
+ version,
+ totalBytesRead,
+ elapsedTimeMs,
+ toMbps(totalBytesRead, elapsedTimeMs),
+ afterReverseSeek));
+
+ assertEquals(testFileLength, totalBytesRead);
+ inputStream.close();
+ return elapsedTimeMs;
+ }
+ }
+
+ private long randomRead(int version, FileSystem fs) throws Exception {
+ assumeHugeFileExists();
+ final long minBytesToRead = 2 * MEGABYTE;
+ Random random = new Random();
+ byte[] buffer = new byte[8 * KILOBYTE];
+ long totalBytesRead = 0;
+ long bytesRead = 0;
+ try(FSDataInputStream inputStream = fs.open(TEST_FILE_PATH)) {
+ ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer();
+ do {
+ bytesRead = inputStream.read(buffer);
+ totalBytesRead += bytesRead;
+ inputStream.seek(random.nextInt(
+ (int) (TEST_FILE_SIZE - buffer.length)));
+ } while (bytesRead > 0 && totalBytesRead < minBytesToRead);
+ long elapsedTimeMs = timer.elapsedTimeMs();
+ inputStream.close();
+ System.out.println(String.format(
+ "v%1$d: totalBytesRead=%2$d, elapsedTimeMs=%3$d, Mbps=%4$.2f",
+ version,
+ totalBytesRead,
+ elapsedTimeMs,
+ toMbps(totalBytesRead, elapsedTimeMs)));
+ assertTrue(minBytesToRead <= totalBytesRead);
+ return elapsedTimeMs;
+ }
+ }
+
+ /**
+ * Calculate megabits per second from the specified values for bytes and
+ * milliseconds.
+ * @param bytes The number of bytes.
+ * @param milliseconds The number of milliseconds.
+ * @return The number of megabits per second.
+ */
+ private static double toMbps(long bytes, long milliseconds) {
+ return bytes / 1000.0 * 8 / milliseconds;
+ }
+
+ private void createTestFile() throws Exception {
+ FileSystem fs = this.getWasbFileSystem();
+
+ if (fs.exists(TEST_FILE_PATH)) {
+ FileStatus status = fs.getFileStatus(TEST_FILE_PATH);
+ if (status.getLen() >= TEST_FILE_SIZE) {
+ return;
+ }
+ }
+
+ byte[] buffer = new byte[CREATE_BUFFER_SIZE];
+ char character = 'a';
+ for (int i = 0; i < buffer.length; i++) {
+ buffer[i] = (byte) character;
+ character = (character == 'z') ? 'a' : (char) ((int) character + 1);
+ }
+
+ System.out.println(("Creating test file {} of size: {} " + TEST_FILE_PATH
+ + TEST_FILE_SIZE));
+ ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer();
+
+ try(FSDataOutputStream outputStream = fs.create(TEST_FILE_PATH)) {
+ int bytesWritten = 0;
+ while (bytesWritten < TEST_FILE_SIZE) {
+ outputStream.write(buffer);
+ bytesWritten += buffer.length;
+ }
+ System.out.println("Closing stream {}" + outputStream);
+ ContractTestUtils.NanoTimer closeTimer
+ = new ContractTestUtils.NanoTimer();
+ outputStream.close();
+ closeTimer.end("time to close() output stream");
+ }
+ timer.end("time to write %d KB", TEST_FILE_SIZE / 1024);
+ testFileLength = fs.getFileStatus(TEST_FILE_PATH).getLen();
+
+ }
+
+ private void assumeHugeFileExists() throws Exception{
+ createTestFile();
+ FileSystem fs = this.getFileSystem();
+ ContractTestUtils.assertPathExists(this.getFileSystem(), "huge file not created", TEST_FILE_PATH);
+ FileStatus status = fs.getFileStatus(TEST_FILE_PATH);
+ ContractTestUtils.assertIsFile(TEST_FILE_PATH, status);
+ assertTrue("File " + TEST_FILE_PATH + " is empty", status.getLen() > 0);
+ }
+
+ private void verifyConsistentReads(FSDataInputStream inputStreamV1,
+ FSDataInputStream inputStreamV2,
+ byte[] bufferV1,
+ byte[] bufferV2) throws IOException {
+ int size = bufferV1.length;
+ final int numBytesReadV1 = inputStreamV1.read(bufferV1, 0, size);
+ assertEquals("Bytes read from wasb stream", size, numBytesReadV1);
+
+ final int numBytesReadV2 = inputStreamV2.read(bufferV2, 0, size);
+ assertEquals("Bytes read from abfs stream", size, numBytesReadV2);
+
+ assertArrayEquals("Mismatch in read data", bufferV1, bufferV2);
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdf5f4c3/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRename.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRename.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRename.java
new file mode 100644
index 0000000..a0e648c
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRename.java
@@ -0,0 +1,152 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs;
+
+import java.io.FileNotFoundException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+/**
+ * Test rename operation.
+ */
+public class ITestAzureBlobFileSystemRename extends DependencyInjectedTest {
+ public ITestAzureBlobFileSystemRename() {
+ super();
+ }
+
+ @Test(expected = FileNotFoundException.class)
+ public void testEnsureFileIsRenamed() throws Exception {
+ final AzureBlobFileSystem fs = this.getFileSystem();
+ fs.create(new Path("testfile"));
+ fs.rename(new Path("testfile"), new Path("testfile2"));
+
+ FileStatus fileStatus = fs.getFileStatus(new Path("testfile2"));
+ assertNotNull(fileStatus);
+
+ fs.getFileStatus(new Path("testfile"));
+ }
+
+ @Test
+ public void testRenameFile() throws Exception {
+ final AzureBlobFileSystem fs = this.getFileSystem();
+ fs.mkdirs(new Path("/testSrc"));
+ fs.create(new Path("/testSrc/file1"));
+
+ fs.rename(new Path("/testSrc"), new Path("/testDst"));
+ FileStatus[] fileStatus = fs.listStatus(new Path("/testDst"));
+ assertNotNull(fileStatus);
+ }
+
+ @Test
+ public void testRenameFileUsingUnicode() throws Exception {
+ final AzureBlobFileSystem fs = this.getFileSystem();
+ //known issue: ListStatus operation to folders/files whose name contains '?' will fail
+ //This is because Auto rest client didn't encode '?' in the uri query parameters
+ String[] folders1 = new String[]{"/%2c%26", "/ÖáΠ⇒", "/A +B", "/A~`!@#$%^&*()-_+={};:'>,<B"};
+ String[] folders2 = new String[]{"/abcÖ⇒123", "/abcÖáΠ⇒123", "/B+ C", "/B~`!@#$%^&*()-_+={};:'>,<C"};
+ String[] files = new String[]{"/%2c%27", "/中文", "/C +D", "/C~`!@#$%^&*()-_+={};:'>,<D"};
+
+ for (int i = 0; i < 4; i++) {
+ Path folderPath1 = new Path(folders1[i]);
+ assertTrue(fs.mkdirs(folderPath1));
+ assertTrue(fs.exists(folderPath1));
+
+ Path filePath = new Path(folders1[i] + files[i]);
+ fs.create(filePath);
+ assertTrue(fs.exists(filePath));
+
+ Path folderPath2 = new Path(folders2[i]);
+ fs.rename(folderPath1, folderPath2);
+ assertFalse(fs.exists(folderPath1));
+ assertTrue(fs.exists(folderPath2));
+
+ FileStatus[] fileStatus = fs.listStatus(folderPath2);
+ assertEquals("/" + fileStatus[0].getPath().getName(), files[i]);
+ assertNotNull(fileStatus);
+ }
+ }
+
+ @Test(expected = FileNotFoundException.class)
+ public void testRenameDirectory() throws Exception {
+ final AzureBlobFileSystem fs = this.getFileSystem();
+ fs.mkdirs(new Path("testDir"));
+ fs.mkdirs(new Path("testDir/test1"));
+ fs.mkdirs(new Path("testDir/test1/test2"));
+ fs.mkdirs(new Path("testDir/test1/test2/test3"));
+
+ Assert.assertTrue(fs.rename(new Path("testDir/test1"), new Path("testDir/test10")));
+ fs.getFileStatus(new Path("testDir/test1"));
+ }
+
+ @Test(expected = FileNotFoundException.class)
+ public void testRenameFirstLevelDirectory() throws Exception {
+ final AzureBlobFileSystem fs = this.getFileSystem();
+ final List<Future> tasks = new ArrayList<>();
+
+ ExecutorService es = Executors.newFixedThreadPool(10);
+ for (int i = 0; i < 1000; i++) {
+ final Path fileName = new Path("/test/" + i);
+ Callable<Void> callable = new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ fs.create(fileName);
+ return null;
+ }
+ };
+
+ tasks.add(es.submit(callable));
+ }
+
+ for (Future<Void> task : tasks) {
+ task.get();
+ }
+
+ es.shutdownNow();
+ fs.rename(new Path("/test"), new Path("/renamedDir"));
+
+ FileStatus[] files = fs.listStatus(new Path("/renamedDir"));
+ Assert.assertEquals(files.length, 1000);
+ fs.getFileStatus(new Path("/test"));
+ }
+
+ @Test
+ public void testRenameRoot() throws Exception {
+ final AzureBlobFileSystem fs = this.getFileSystem();
+ boolean renamed = fs.rename(new Path("/"), new Path("/ddd"));
+ assertFalse(renamed);
+
+ renamed = fs.rename(new Path(fs.getUri().toString() + "/"), new Path(fs.getUri().toString() + "/s"));
+ assertFalse(renamed);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdf5f4c3/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemInitialization.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemInitialization.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemInitialization.java
new file mode 100644
index 0000000..aa30a85
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemInitialization.java
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs;
+
+import java.net.URI;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes;
+import org.apache.hadoop.fs.azurebfs.contracts.services.AbfsHttpService;
+import org.apache.hadoop.fs.azurebfs.services.AbfsServiceProviderImpl;
+
+import static org.mockito.Matchers.anyObject;
+import static org.mockito.Mockito.doReturn;
+
+/**
+ * Test AzureBlobFileSystem initialization.
+ */
+public class ITestFileSystemInitialization extends DependencyInjectedTest {
+ public ITestFileSystemInitialization() {
+ super();
+
+ this.getMockServiceInjector().removeProvider(AbfsHttpService.class);
+ this.getMockServiceInjector().replaceInstance(AbfsHttpService.class, Mockito.mock(AbfsHttpService.class));
+ }
+
+ @Test
+ public void ensureAzureBlobFileSystemIsInitialized() throws Exception {
+ doReturn(new FileStatus(0, true, 0, 0, 0, new Path("/blah")))
+ .when(AbfsServiceProviderImpl.instance().get(AbfsHttpService.class))
+ .getFileStatus((AzureBlobFileSystem) anyObject(), (Path) anyObject());
+
+ final FileSystem fs = FileSystem.get(this.getConfiguration());
+ final String accountName = this.getAccountName();
+ final String filesystem = this.getFileSystemName();
+
+ Assert.assertEquals(fs.getUri(), new URI(FileSystemUriSchemes.ABFS_SCHEME, filesystem + "@" + accountName, null, null, null));
+ Assert.assertNotNull(fs.getWorkingDirectory());
+ }
+
+ @Test
+ public void ensureSecureAzureBlobFileSystemIsInitialized() throws Exception {
+ doReturn(new FileStatus(0, true, 0, 0, 0, new Path("/blah")))
+ .when(AbfsServiceProviderImpl.instance().get(AbfsHttpService.class))
+ .getFileStatus((AzureBlobFileSystem) anyObject(), (Path) anyObject());
+
+ final String accountName = this.getAccountName();
+ final String filesystem = this.getFileSystemName();
+ final URI defaultUri = new URI(FileSystemUriSchemes.ABFS_SECURE_SCHEME, filesystem + "@" + accountName, null, null, null);
+ this.getConfiguration().set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri.toString());
+
+ final FileSystem fs = FileSystem.get(this.getConfiguration());
+ Assert.assertEquals(fs.getUri(), new URI(FileSystemUriSchemes.ABFS_SECURE_SCHEME, filesystem + "@" + accountName, null, null, null));
+ Assert.assertNotNull(fs.getWorkingDirectory());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdf5f4c3/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemRegistration.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemRegistration.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemRegistration.java
new file mode 100644
index 0000000..a55599b
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemRegistration.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs;
+
+import java.net.URI;
+
+import org.apache.hadoop.fs.azurebfs.services.AbfsServiceProviderImpl;
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import org.apache.hadoop.fs.AbstractFileSystem;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes;
+import org.apache.hadoop.fs.azurebfs.contracts.services.AbfsHttpService;
+
+import static org.mockito.Matchers.anyObject;
+import static org.mockito.Mockito.doReturn;
+
+/**
+ * Test AzureBlobFileSystem registration.
+ */
+public class ITestFileSystemRegistration extends DependencyInjectedTest {
+ public ITestFileSystemRegistration() throws Exception {
+ super();
+
+ this.getMockServiceInjector().removeProvider(AbfsHttpService.class);
+ this.getMockServiceInjector().replaceInstance(AbfsHttpService.class, Mockito.mock(AbfsHttpService.class));
+ }
+
+ @Test
+ public void ensureAzureBlobFileSystemIsDefaultFileSystem() throws Exception {
+ doReturn(new FileStatus(0, true, 0, 0, 0, new Path("/blah")))
+ .when(AbfsServiceProviderImpl.instance().get(AbfsHttpService.class))
+ .getFileStatus((AzureBlobFileSystem) anyObject(), (Path) anyObject());
+
+ FileSystem fs = FileSystem.get(this.getConfiguration());
+ Assert.assertTrue(fs instanceof AzureBlobFileSystem);
+
+ AbstractFileSystem afs = FileContext.getFileContext(this.getConfiguration()).getDefaultFileSystem();
+ Assert.assertTrue(afs instanceof Abfs);
+ }
+
+ @Test
+ public void ensureSecureAzureBlobFileSystemIsDefaultFileSystem() throws Exception {
+ doReturn(new FileStatus(0, true, 0, 0, 0, new Path("/blah")))
+ .when(AbfsServiceProviderImpl.instance().get(AbfsHttpService.class))
+ .getFileStatus((AzureBlobFileSystem) anyObject(), (Path) anyObject());
+
+ final String accountName = this.getAccountName();
+ final String filesystem = this.getFileSystemName();
+
+ final URI defaultUri = new URI(FileSystemUriSchemes.ABFS_SECURE_SCHEME, filesystem + "@" + accountName, null, null, null);
+ this.getConfiguration().set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri.toString());
+
+ FileSystem fs = FileSystem.get(this.getConfiguration());
+ Assert.assertTrue(fs instanceof SecureAzureBlobFileSystem);
+
+ AbstractFileSystem afs = FileContext.getFileContext(this.getConfiguration()).getDefaultFileSystem();
+ Assert.assertTrue(afs instanceof Abfss);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdf5f4c3/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java
new file mode 100644
index 0000000..7010e74
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java
@@ -0,0 +1,202 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.azurebfs;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.azure.NativeAzureFileSystem;
+
+import org.junit.Assert;
+import org.junit.Assume;
+import org.junit.Test;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+
+import static junit.framework.TestCase.assertEquals;
+import static junit.framework.TestCase.assertFalse;
+import static junit.framework.TestCase.assertTrue;
+
+/**
+ * Test compatibility between ABFS client and WASB client.
+ */
+public class ITestWasbAbfsCompatibility extends DependencyInjectedTest {
+ private static final String WASB_TEST_CONTEXT = "wasb test file";
+ private static final String ABFS_TEST_CONTEXT = "abfs test file";
+ private static final String TEST_CONTEXT = "THIS IS FOR TEST";
+
+ public ITestWasbAbfsCompatibility() throws Exception {
+ super();
+
+ Assume.assumeFalse(this.isEmulator());
+ }
+
+ @Test
+ public void testListFileStatus() throws Exception {
+ // crate file using abfs
+ AzureBlobFileSystem fs = this.getFileSystem();
+ NativeAzureFileSystem wasb = this.getWasbFileSystem();
+
+ Path path1 = new Path("/testfiles/~12/!008/3/abFsTestfile");
+ FSDataOutputStream abfsStream = fs.create(path1, true);
+ abfsStream.write(ABFS_TEST_CONTEXT.getBytes());
+ abfsStream.flush();
+ abfsStream.hsync();
+ abfsStream.close();
+
+ // create file using wasb
+ Path path2 = new Path("/testfiles/~12/!008/3/nativeFsTestfile");
+ System.out.println(wasb.getUri());
+ FSDataOutputStream nativeFsStream = wasb.create(path2, true);
+ nativeFsStream.write(WASB_TEST_CONTEXT.getBytes());
+ nativeFsStream.flush();
+ nativeFsStream.hsync();
+ nativeFsStream.close();
+ // list file using abfs and wasb
+ FileStatus[] abfsFileStatus = fs.listStatus(new Path("/testfiles/~12/!008/3/"));
+ FileStatus[] nativeFsFileStatus = wasb.listStatus(new Path("/testfiles/~12/!008/3/"));
+
+ assertEquals(2, abfsFileStatus.length);
+ assertEquals(2, nativeFsFileStatus.length);
+ }
+
+ @Test
+ public void testReadFile() throws Exception {
+ boolean[] createFileWithAbfs = new boolean[]{false, true, false, true};
+ boolean[] readFileWithAbfs = new boolean[]{false, true, true, false};
+
+ AzureBlobFileSystem abfs = this.getFileSystem();
+ NativeAzureFileSystem wasb = this.getWasbFileSystem();
+
+ FileSystem fs;
+ BufferedReader br = null;
+ for (int i = 0; i< 4; i++) {
+ try {
+ Path path = new Path("/testfiles/~12/!008/testfile" + i);
+ if (createFileWithAbfs[i]) {
+ fs = abfs;
+ } else {
+ fs = wasb;
+ }
+
+ // Write
+ FSDataOutputStream nativeFsStream = fs.create(path, true);
+ nativeFsStream.write(TEST_CONTEXT.getBytes());
+ nativeFsStream.flush();
+ nativeFsStream.hsync();
+ nativeFsStream.close();
+
+ // Check file status
+ assertEquals(true, fs.exists(path));
+ assertEquals(false, fs.getFileStatus(path).isDirectory());
+
+ // Read
+ if (readFileWithAbfs[i]) {
+ fs = abfs;
+ } else {
+ fs = wasb;
+ }
+ FSDataInputStream inputStream = fs.open(path);
+ br = new BufferedReader(new InputStreamReader(fs.open(path)));
+ String line = br.readLine();
+ assertEquals(TEST_CONTEXT, line);
+
+ // Remove file
+ fs.delete(path, true);
+ assertFalse(fs.exists(path));
+ } catch (Exception e) {
+ e.printStackTrace();
+ } finally {
+ if (br != null) {
+ br.close();
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testDir() throws Exception {
+ boolean[] createDirWithAbfs = new boolean[]{false, true, false, true};
+ boolean[] readDirWithAbfs = new boolean[]{false, true, true, false};
+
+ AzureBlobFileSystem abfs = this.getFileSystem();
+ NativeAzureFileSystem wasb = this.getWasbFileSystem();
+
+ FileSystem fs;
+ for (int i = 0; i < 4; i++) {
+ Path path = new Path("/testDir/t" + i);
+ //create
+ if (createDirWithAbfs[i]) {
+ fs = abfs;
+ } else {
+ fs = wasb;
+ }
+ assertTrue(fs.mkdirs(path));
+ //check
+ assertTrue(fs.exists(path));
+ //read
+ if (readDirWithAbfs[i]) {
+ fs = abfs;
+ } else {
+ fs = wasb;
+ }
+ assertTrue(fs.exists(path));
+ FileStatus dirStatus = fs.getFileStatus(path);
+ assertTrue(dirStatus.isDirectory());
+ fs.delete(path, true);
+ assertFalse(fs.exists(path));
+ }
+ }
+
+
+ @Test
+ public void testUrlConversion(){
+ String abfsUrl = "abfs://abcde-1111-1111-1111-1111@xxxx.dfs.xxx.xxx.xxxx.xxxx";
+ String wabsUrl = "wasb://abcde-1111-1111-1111-1111@xxxx.blob.xxx.xxx.xxxx.xxxx";
+ Assert.assertEquals(abfsUrl, wasbUrlToAbfsUrl(wabsUrl));
+ Assert.assertEquals(wabsUrl, abfsUrlToWasbUrl(abfsUrl));
+ }
+
+ @Test
+ public void testSetWorkingDirectory() throws Exception {
+ //create folders
+ AzureBlobFileSystem abfs = this.getFileSystem();
+ NativeAzureFileSystem wasb = this.getWasbFileSystem();
+
+ assertTrue(abfs.mkdirs(new Path("/d1/d2/d3/d4")));
+
+ //set working directory to path1
+ Path path1 = new Path("/d1/d2");
+ wasb.setWorkingDirectory(path1);
+ abfs.setWorkingDirectory(path1);
+ assertEquals(path1, wasb.getWorkingDirectory());
+ assertEquals(path1, abfs.getWorkingDirectory());
+
+ //set working directory to path2
+ Path path2 = new Path("d3/d4");
+ wasb.setWorkingDirectory(path2);
+ abfs.setWorkingDirectory(path2);
+
+ Path path3 = new Path("/d1/d2/d3/d4");
+ assertEquals(path3, wasb.getWorkingDirectory());
+ assertEquals(path3, abfs.getWorkingDirectory());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdf5f4c3/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java
new file mode 100644
index 0000000..4b44765
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.constants;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Responsible to keep all the Azure Blob File System configurations keys in Hadoop configuration file.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public final class TestConfigurationKeys {
+ public static final String FS_AZURE_TEST_ACCOUNT_NAME = "fs.azure.test.account.name";
+ public static final String FS_AZURE_TEST_ACCOUNT_KEY_PREFIX = "fs.azure.test.account.key.";
+ public static final String FS_AZURE_TEST_HOST_NAME = "fs.azure.test.host.name";
+ public static final String FS_AZURE_TEST_HOST_PORT = "fs.azure.test.host.port";
+ public static final String FS_AZURE_CONTRACT_TEST_URI = "fs.contract.test.fs.abfs";
+
+ private TestConfigurationKeys() {}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdf5f4c3/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/package-info.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/package-info.java
new file mode 100644
index 0000000..109f887
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+package org.apache.hadoop.fs.azurebfs.constants;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdf5f4c3/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/DependencyInjectedContractTest.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/DependencyInjectedContractTest.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/DependencyInjectedContractTest.java
new file mode 100644
index 0000000..5fc81ce
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/DependencyInjectedContractTest.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.contract;
+
+import java.net.URI;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.azurebfs.DependencyInjectedTest;
+import org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes;
+import org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys;
+
+/**
+ * Dependency inject for ABFS contract tests.
+ */
+public class DependencyInjectedContractTest extends DependencyInjectedTest {
+ private final URI testUri;
+
+ public DependencyInjectedContractTest(final boolean secure) throws Exception {
+ this(secure, true);
+ }
+
+ public DependencyInjectedContractTest(final boolean secure, final boolean useExistedFileSystem) throws Exception{
+ super(secure);
+ if (useExistedFileSystem) {
+ Configuration configuration = getConfiguration();
+ String testUrl = configuration.get(TestConfigurationKeys.FS_AZURE_CONTRACT_TEST_URI);
+
+ if (secure) {
+ testUrl = testUrl.replaceFirst(FileSystemUriSchemes.ABFS_SCHEME, FileSystemUriSchemes.ABFS_SECURE_SCHEME);
+ }
+ updateTestUrl(testUrl);
+
+ this.testUri = new URI(testUrl);
+ //Get container for contract tests
+ configuration.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, this.testUri.toString());
+ String[] splitAuthority = this.testUri.getAuthority().split("\\@");
+ updateFileSystemName(splitAuthority[0]);
+ } else {
+ this.testUri = new URI(super.getTestUrl());
+ }
+ }
+
+ public Configuration getConfiguration() {
+ return super.getConfiguration();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdf5f4c3/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContract.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContract.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContract.java
new file mode 100644
index 0000000..7f7a09a
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContract.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes;
+import org.apache.hadoop.fs.azurebfs.utils.UriUtils;
+import org.apache.hadoop.fs.contract.AbstractBondedFSContract;
+
+/**
+ * Azure BlobFileSystem Contract. Test paths are created using any maven fork
+ * identifier, if defined. This guarantees paths unique to tests
+ * running in parallel.
+ */
+public class ITestAbfsFileSystemContract extends AbstractBondedFSContract {
+
+ public static final String CONTRACT_XML = "abfs.xml";
+ private final boolean isSecure;
+
+ protected ITestAbfsFileSystemContract(final Configuration conf, boolean secure) {
+ super(conf);
+ //insert the base features
+ addConfResource(CONTRACT_XML);
+ this.isSecure = secure;
+ }
+
+ @Override
+ public String getScheme() {
+ return isSecure ? FileSystemUriSchemes.ABFS_SECURE_SCHEME : FileSystemUriSchemes.ABFS_SCHEME;
+ }
+
+ @Override
+ public Path getTestPath() {
+ Path path = new Path(UriUtils.generateUniqueTestPath());
+ return path;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdf5f4c3/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractAppend.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractAppend.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractAppend.java
new file mode 100644
index 0000000..d4cca14
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractAppend.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.contract;
+
+import java.util.Arrays;
+
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractAppendTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+import static org.apache.hadoop.fs.contract.ContractTestUtils.skip;
+
+/**
+ * Contract test for open operation.
+ */
+@RunWith(Parameterized.class)
+public class ITestAbfsFileSystemContractAppend extends AbstractContractAppendTest {
+ @Parameterized.Parameters(name = "SecureMode={0}")
+ public static Iterable<Object[]> secure() {
+ return Arrays.asList(new Object[][] { {true}, {false} });
+ }
+
+ private final boolean isSecure;
+ private final DependencyInjectedContractTest dependencyInjectedContractTest;
+
+ public ITestAbfsFileSystemContractAppend(final boolean secure) throws Exception {
+ this.isSecure = secure;
+ dependencyInjectedContractTest = new DependencyInjectedContractTest(this.isSecure);
+ }
+
+ @Override
+ public void setup() throws Exception {
+ dependencyInjectedContractTest.initialize();
+ super.setup();
+ }
+
+ @Override
+ protected Configuration createConfiguration() {
+ return this.dependencyInjectedContractTest.getConfiguration();
+ }
+
+ @Override
+ protected AbstractFSContract createContract(final Configuration conf) {
+ return new ITestAbfsFileSystemContract(conf, this.isSecure);
+ }
+
+ @Override
+ public void testRenameFileBeingAppended() throws Throwable {
+ skip("Skipping as renaming an opened file is not supported");
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdf5f4c3/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractConcat.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractConcat.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractConcat.java
new file mode 100644
index 0000000..4f724e2
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractConcat.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.azurebfs.contract;
+
+import java.util.Arrays;
+
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractConcatTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * Contract test for concat operation.
+ */
+@RunWith(Parameterized.class)
+public class ITestAbfsFileSystemContractConcat extends AbstractContractConcatTest{
+ @Parameterized.Parameters(name = "SecureMode={0}")
+ public static Iterable<Object[]> secure() {
+ return Arrays.asList(new Object[][] { {true}, {false} });
+ }
+
+ private final boolean isSecure;
+ private final DependencyInjectedContractTest dependencyInjectedContractTest;
+
+ public ITestAbfsFileSystemContractConcat(final boolean secure) throws Exception {
+ this.isSecure = secure;
+ dependencyInjectedContractTest = new DependencyInjectedContractTest(isSecure);
+ }
+
+ @Override
+ public void setup() throws Exception {
+ dependencyInjectedContractTest.initialize();
+ super.setup();
+ }
+
+ @Override
+ protected Configuration createConfiguration() {
+ return this.dependencyInjectedContractTest.getConfiguration();
+ }
+
+ @Override
+ protected AbstractFSContract createContract(final Configuration conf) {
+ return new ITestAbfsFileSystemContract(conf, this.isSecure);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdf5f4c3/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractCreate.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractCreate.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractCreate.java
new file mode 100644
index 0000000..16b959f
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractCreate.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.contract;
+
+import java.util.Arrays;
+
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractCreateTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * Contract test for create operation.
+ */
+@RunWith(Parameterized.class)
+public class ITestAbfsFileSystemContractCreate extends AbstractContractCreateTest{
+ @Parameterized.Parameters(name = "SecureMode={0}")
+ public static Iterable<Object[]> secure() {
+ return Arrays.asList(new Object[][] { {true}, {false} });
+ }
+
+ private final boolean isSecure;
+ private final DependencyInjectedContractTest dependencyInjectedContractTest;
+
+ public ITestAbfsFileSystemContractCreate(final boolean secure) throws Exception {
+ this.isSecure = secure;
+ dependencyInjectedContractTest = new DependencyInjectedContractTest(this.isSecure);
+ }
+
+ @Override
+ public void setup() throws Exception {
+ dependencyInjectedContractTest.initialize();
+ super.setup();
+ }
+
+ @Override
+ protected Configuration createConfiguration() {
+ return this.dependencyInjectedContractTest.getConfiguration();
+ }
+
+ @Override
+ protected AbstractFSContract createContract(final Configuration conf) {
+ return new ITestAbfsFileSystemContract(conf, this.isSecure);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdf5f4c3/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractDelete.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractDelete.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractDelete.java
new file mode 100644
index 0000000..fabd3273
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractDelete.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.contract;
+
+import java.util.Arrays;
+
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractDeleteTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * Contract test for delete operation.
+ */
+@RunWith(Parameterized.class)
+public class ITestAbfsFileSystemContractDelete extends AbstractContractDeleteTest {
+ @Parameterized.Parameters(name = "SecureMode={0}")
+ public static Iterable<Object[]> secure() {
+ return Arrays.asList(new Object[][] { {true}, {false} });
+ }
+
+ private final boolean isSecure;
+ private final DependencyInjectedContractTest dependencyInjectedContractTest;
+
+ public ITestAbfsFileSystemContractDelete(final boolean secure) throws Exception {
+ this.isSecure = secure;
+ dependencyInjectedContractTest = new DependencyInjectedContractTest(isSecure);
+ }
+
+ @Override
+ public void setup() throws Exception {
+ dependencyInjectedContractTest.initialize();
+ super.setup();
+ }
+
+ @Override
+ protected Configuration createConfiguration() {
+ return this.dependencyInjectedContractTest.getConfiguration();
+ }
+
+ @Override
+ protected AbstractFSContract createContract(final Configuration conf) {
+ return new ITestAbfsFileSystemContract(conf, this.isSecure);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdf5f4c3/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractDistCp.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractDistCp.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractDistCp.java
new file mode 100644
index 0000000..a1360e4
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractDistCp.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.tools.contract.AbstractContractDistCpTest;
+
+/**
+ * Contract test for distCp operation.
+ */
+public class ITestAbfsFileSystemContractDistCp extends AbstractContractDistCpTest {
+ private final DependencyInjectedContractTest dependencyInjectedContractTest;
+
+ public ITestAbfsFileSystemContractDistCp() throws Exception {
+ dependencyInjectedContractTest = new DependencyInjectedContractTest(false);
+ }
+
+ @Override
+ public void setup() throws Exception {
+ dependencyInjectedContractTest.initialize();
+ super.setup();
+ }
+
+ @Override
+ protected ITestAbfsFileSystemContract createContract(Configuration conf) {
+ return new ITestAbfsFileSystemContract(conf, false);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdf5f4c3/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractGetFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractGetFileStatus.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractGetFileStatus.java
new file mode 100644
index 0000000..5bb41ad
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractGetFileStatus.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.azurebfs.contract;
+
+import java.util.Arrays;
+
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractGetFileStatusTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * Contract test for getFileStatus operation.
+ */
+@RunWith(Parameterized.class)
+public class ITestAbfsFileSystemContractGetFileStatus extends AbstractContractGetFileStatusTest {
+ @Parameterized.Parameters(name = "SecureMode={0}")
+ public static Iterable<Object[]> secure() {
+ return Arrays.asList(new Object[][] { {true}, {false} });
+ }
+
+ private final boolean isSecure;
+ private final DependencyInjectedContractTest dependencyInjectedContractTest;
+
+ public ITestAbfsFileSystemContractGetFileStatus(final boolean secure) throws Exception {
+ this.isSecure = secure;
+ dependencyInjectedContractTest = new DependencyInjectedContractTest(isSecure);
+ }
+
+ @Override
+ public void setup() throws Exception {
+ dependencyInjectedContractTest.initialize();
+ super.setup();
+ }
+
+ @Override
+ protected Configuration createConfiguration() {
+ return this.dependencyInjectedContractTest.getConfiguration();
+ }
+
+ @Override
+ protected AbstractFSContract createContract(final Configuration conf) {
+ return new ITestAbfsFileSystemContract(conf, this.isSecure);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdf5f4c3/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractMkdir.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractMkdir.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractMkdir.java
new file mode 100644
index 0000000..9d732d5
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractMkdir.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.contract;
+
+import java.util.Arrays;
+
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractMkdirTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * Contract test for mkdir operation.
+ */
+@RunWith(Parameterized.class)
+public class ITestAbfsFileSystemContractMkdir extends AbstractContractMkdirTest {
+ @Parameterized.Parameters(name = "SecureMode={0}")
+ public static Iterable<Object[]> secure() {
+ return Arrays.asList(new Object[][] { {true}, {false} });
+ }
+
+ private final boolean isSecure;
+ private final DependencyInjectedContractTest dependencyInjectedContractTest;
+
+ public ITestAbfsFileSystemContractMkdir(final boolean secure) throws Exception {
+ this.isSecure = secure;
+ dependencyInjectedContractTest = new DependencyInjectedContractTest(secure);
+ }
+
+ @Override
+ public void setup() throws Exception {
+ dependencyInjectedContractTest.initialize();
+ super.setup();
+ }
+
+ @Override
+ protected Configuration createConfiguration() {
+ return this.dependencyInjectedContractTest.getConfiguration();
+ }
+
+ @Override
+ protected AbstractFSContract createContract(final Configuration conf) {
+ return new ITestAbfsFileSystemContract(conf, this.isSecure);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdf5f4c3/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractOpen.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractOpen.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractOpen.java
new file mode 100644
index 0000000..a71149b
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractOpen.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.contract;
+
+import java.util.Arrays;
+
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractOpenTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * Contract test for open operation.
+ */
+@RunWith(Parameterized.class)
+public class ITestAbfsFileSystemContractOpen extends AbstractContractOpenTest {
+ @Parameterized.Parameters(name = "SecureMode={0}")
+ public static Iterable<Object[]> secure() {
+ return Arrays.asList(new Object[][] { {true}, {false} });
+ }
+
+ private final boolean isSecure;
+ private final DependencyInjectedContractTest dependencyInjectedContractTest;
+
+ public ITestAbfsFileSystemContractOpen(final boolean secure) throws Exception {
+ this.isSecure = secure;
+ dependencyInjectedContractTest = new DependencyInjectedContractTest(this.isSecure);
+ }
+
+ @Override
+ public void setup() throws Exception {
+ dependencyInjectedContractTest.initialize();
+ super.setup();
+ }
+
+ @Override
+ protected Configuration createConfiguration() {
+ return this.dependencyInjectedContractTest.getConfiguration();
+ }
+
+ @Override
+ protected AbstractFSContract createContract(final Configuration conf) {
+ return new ITestAbfsFileSystemContract(conf, this.isSecure);
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org