You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ka...@apache.org on 2016/11/17 02:02:10 UTC
[08/50] hadoop git commit: HADOOP-13037. Refactor Azure Data Lake
Store as an independent FileSystem. Contributed by Vishwajeet Dusane
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5c61ad24/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlDifferentSizeWritesLive.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlDifferentSizeWritesLive.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlDifferentSizeWritesLive.java
index b712995..8f53400 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlDifferentSizeWritesLive.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlDifferentSizeWritesLive.java
@@ -73,7 +73,7 @@ public class TestAdlDifferentSizeWritesLive {
Path path = new Path("/test/dataIntegrityCheck");
FileSystem fs = null;
try {
- fs = AdlStorageConfiguration.createAdlStorageConnector();
+ fs = AdlStorageConfiguration.createStorageConnector();
} catch (URISyntaxException e) {
throw new IllegalStateException("Can not initialize ADL FileSystem. "
+ "Please check fs.defaultFS property.", e);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5c61ad24/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlFileSystemContractLive.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlFileSystemContractLive.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlFileSystemContractLive.java
index e007c4e..0df7d05 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlFileSystemContractLive.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlFileSystemContractLive.java
@@ -34,7 +34,7 @@ public class TestAdlFileSystemContractLive extends FileSystemContractBaseTest {
@Override
protected void setUp() throws Exception {
- adlStore = AdlStorageConfiguration.createAdlStorageConnector();
+ adlStore = AdlStorageConfiguration.createStorageConnector();
if (AdlStorageConfiguration.isContractTestEnabled()) {
fs = adlStore;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5c61ad24/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlReadLive.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlReadLive.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlReadLive.java
deleted file mode 100644
index f1d909c..0000000
--- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlReadLive.java
+++ /dev/null
@@ -1,342 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.hadoop.fs.adl.live;
-
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.net.URISyntaxException;
-import java.util.Random;
-import java.util.UUID;
-
-/**
- * Verify different data segment size read from the file to ensure the
- * integrity and order of the data over
- * BufferManger and BatchByteArrayInputStream implementation.
- */
-public class TestAdlReadLive {
- private String expectedData = "1234567890abcdefghijklmnopqrstuvwxyz";
-
- @Before
- public void setup() throws Exception {
- org.junit.Assume
- .assumeTrue(AdlStorageConfiguration.isContractTestEnabled());
- }
-
- private FileSystem getFileSystem() throws IOException, URISyntaxException {
- return AdlStorageConfiguration.createAdlStorageConnector();
- }
-
- private void setupFile(Path path) throws IOException, URISyntaxException {
- setupFile(path, expectedData);
- }
-
- private void setupFile(Path path, String data)
- throws IOException, URISyntaxException {
- expectedData = data;
- FileSystem fs = getFileSystem();
- fs.delete(path, true);
- FSDataOutputStream fdis = fs.create(path);
- fdis.writeBytes(expectedData);
- fdis.close();
- fs.listStatus(path.getParent());
- long actualLen = fs.getFileStatus(path).getLen();
- long expectedLen = expectedData.length();
- System.out.println(
- " Length of file : " + fs.getFileStatus(path).getLen() + " " + fs
- .getUri());
- Assert.assertEquals(expectedLen, actualLen);
- }
-
- @Test
- public void
- testOpenReadMoreThanAvailableBufferCrashFixIndexOutOfBoundsException()
- throws Throwable {
- Path path = new Path("/test1");
- FileSystem fs = getFileSystem();
- setupFile(path);
-
- if (fs.exists(path)) {
- Assert.assertTrue(fs.delete(path, true));
- }
-
- FSDataOutputStream outputStream = fs.create(path);
- final byte[] data = new byte[24 * 1024 * 1024];
- Random ran = new Random();
- ran.nextBytes(data);
- outputStream.write(data);
-
- FSDataInputStream bb = fs.open(path);
- byte[] expected = new byte[4 * 1024 * 1024];
- bb.read();
- bb.readFully(16711581, expected, 33,
- 65640); // BugFix : Was causing crash IndexOutOfBoundsException
- bb.seek(16711581);
- bb.readFully(16711576, expected, 33, 65640);
- bb.readFully(16711578, expected, 33, 65640);
- bb.readFully(16711580, expected, 33, 65640);
- bb.readFully(16711576, expected, 0, expected.length);
- bb.seek(0);
- expected = new byte[134144];
- while (bb.read() != -1){
- continue;
- }
- bb.readFully(0, data, 0, data.length);
- }
-
- @Test
- public void readNullData() throws IOException, URISyntaxException {
- String data = "SPL \u0001Lorg.apache.hadoop.examples.terasort"
- + ".TeraGen$RangeInputFormat$RangeInputSplit \u008DLK@Lorg.apache"
- + ".hadoop.examples.terasort"
- + ".TeraGen$RangeInputFormat$RangeInputSplit\u008DLK@\u008DLK@";
- Path path = new Path("/test4");
- FileSystem fs = this.getFileSystem();
- setupFile(path, data);
- FSDataInputStream bb = fs.open(path);
- int i = 0;
- String actualData = new String();
- System.out.println("Data Length :" + expectedData.length());
- byte[] arr = new byte[data.length()];
- bb.readFully(0, arr);
- actualData = new String(arr);
- System.out.println(" Data : " + actualData);
- Assert.assertEquals(actualData.length(), expectedData.length());
-
- arr = new byte[data.length() - 7];
- bb.readFully(7, arr);
- actualData = new String(arr);
- Assert.assertEquals(actualData.length(), expectedData.length() - 7);
- bb.close();
- }
-
- @Test
- public void readTest() throws IOException, URISyntaxException {
- Path path = new Path("/test4");
- FileSystem fs = this.getFileSystem();
- setupFile(path);
- FSDataInputStream bb = fs.open(path);
- int i = 0;
- String actualData = new String();
- while (true) {
- int c = bb.read();
- if (c < 0) {
- break;
- }
- actualData += (char) c;
- }
-
- byte[] b = new byte[100];
- System.out.println(bb.read(b, 9, 91));
- System.out.println(bb.read());
- System.out.println(bb.read());
- System.out.println(bb.read());
- System.out.println(bb.read());
- System.out.println(bb.read());
- System.out.println(bb.read());
-
- bb.close();
- Assert.assertEquals(actualData, expectedData);
-
- for (int j = 0; j < 100; ++j) {
- fs = this.getFileSystem();
- fs.exists(new Path("/test" + j));
- }
- }
-
- @Test
- public void readByteTest() throws IOException, URISyntaxException {
- Path path = new Path("/test3");
- FileSystem fs = this.getFileSystem();
- setupFile(path);
- FSDataInputStream bb = fs.open(path);
- int i = 0;
- byte[] data = new byte[expectedData.length()];
- int readByte = bb.read(data);
- bb.close();
- Assert.assertEquals(readByte, expectedData.length());
- Assert.assertEquals(new String(data), expectedData);
- }
-
- @Test
- public void readByteFullyTest() throws IOException, URISyntaxException {
- Path path = new Path("/test2");
- FileSystem fs = this.getFileSystem();
- setupFile(path);
- FSDataInputStream bb = fs.open(path);
- int i = 0;
- byte[] data = new byte[expectedData.length()];
- bb.readFully(data);
- bb.close();
- Assert.assertEquals(new String(data), expectedData);
-
- bb = fs.open(path);
- bb.readFully(data, 0, data.length);
- bb.close();
- Assert.assertEquals(new String(data), expectedData);
- }
-
- @Test
- public void readCombinationTest() throws IOException, URISyntaxException {
- Path path = new Path("/test1");
- FileSystem fs = this.getFileSystem();
- setupFile(path);
- FSDataInputStream bb = fs.open(path);
- int i = 0;
- byte[] data = new byte[5];
- int readByte = bb.read(data);
- Assert.assertEquals(new String(data), expectedData.substring(0, 5));
-
- bb.readFully(data, 0, data.length);
- Assert.assertEquals(new String(data), expectedData.substring(5, 10));
- bb.close();
- bb = fs.open(path);
- bb.readFully(5, data, 0, data.length);
- Assert.assertEquals(new String(data), expectedData.substring(5, 10));
-
- bb.read(data);
- Assert.assertEquals(new String(data), expectedData.substring(0, 5));
- bb.close();
- bb = fs.open(path);
- bb.read(new byte[100]);
- bb.close();
- }
-
- @Test
- public void readMultiSeekTest() throws IOException, URISyntaxException {
- final Path path = new Path(
- "/delete14/" + UUID.randomUUID().toString().replaceAll("-", ""));
- FileSystem fs = this.getFileSystem();
-
- final byte[] actualData = new byte[3267397];
- Random ran = new Random();
- ran.nextBytes(actualData);
- byte[] testData = null;
-
- fs.delete(path, true);
- FSDataOutputStream os = fs.create(path);
- os.write(actualData);
- os.close();
-
- FSDataInputStream bb = fs.open(path);
- byte[] data = new byte[16384];
- bb.readFully(3251013, data, 0, 16384);
- testData = new byte[16384];
- System.arraycopy(actualData, 3251013, testData, 0, 16384);
- Assert.assertArrayEquals(testData, data);
-
- data = new byte[1921];
- bb.readFully(3265476, data, 0, 1921);
- testData = new byte[1921];
- System.arraycopy(actualData, 3265476, testData, 0, 1921);
- Assert.assertArrayEquals(testData, data);
-
- data = new byte[3267394];
- bb.readFully(3, data, 0, 3267394);
- testData = new byte[3267394];
- System.arraycopy(actualData, 3, testData, 0, 3267394);
- Assert.assertArrayEquals(testData, data);
-
- data = new byte[3266943];
- bb.readFully(454, data, 0, 3266943);
- testData = new byte[3266943];
- System.arraycopy(actualData, 454, testData, 0, 3266943);
- Assert.assertArrayEquals(testData, data);
-
- data = new byte[3265320];
- bb.readFully(2077, data, 0, 3265320);
- testData = new byte[3265320];
- System.arraycopy(actualData, 2077, testData, 0, 3265320);
- Assert.assertArrayEquals(testData, data);
-
- bb.close();
-
- bb = fs.open(path);
-
- data = new byte[3263262];
- bb.readFully(4135, data, 0, 3263262);
- testData = new byte[3263262];
- System.arraycopy(actualData, 4135, testData, 0, 3263262);
- Assert.assertArrayEquals(testData, data);
-
- data = new byte[2992591];
- bb.readFully(274806, data, 0, 2992591);
- testData = new byte[2992591];
- System.arraycopy(actualData, 274806, testData, 0, 2992591);
- Assert.assertArrayEquals(testData, data);
-
- data = new byte[1985665];
- bb.readFully(1281732, data, 0, 1985665);
- testData = new byte[1985665];
- System.arraycopy(actualData, 1281732, testData, 0, 1985665);
- Assert.assertArrayEquals(testData, data);
-
- data = new byte[3267394];
- try {
- bb.readFully(2420207, data, 0, 3267394);
- Assert.fail("EOF expected");
- } catch (IOException e) {
- }
-
- bb.close();
- }
-
- @Test
- public void allASCIICharTest() throws IOException, URISyntaxException {
- final Path path = new Path(
- "/delete14/" + UUID.randomUUID().toString().replaceAll("-", ""));
- FileSystem fs = this.getFileSystem();
- final byte[] actualData = new byte[127];
- for (byte i = 0; i < 127; ++i) {
- actualData[i] = i;
- }
-
- fs.delete(path, true);
- FSDataOutputStream os = fs.create(path);
- os.write(actualData);
- os.close();
-
- FSDataInputStream bb = fs.open(path);
- byte[] data = new byte[127];
-
- bb.readFully(0, data, 0, data.length);
- bb.close();
- Assert.assertArrayEquals(data, actualData);
-
- bb = fs.open(path);
- int byteRead = 1;
- while (bb.read() != -1) {
- byteRead++;
- }
-
- bb.seek(0);
- byteRead = 1;
- while (bb.read() != -1) {
- byteRead++;
- }
- bb.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5c61ad24/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlWebHdfsFileContextCreateMkdirLive.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlWebHdfsFileContextCreateMkdirLive.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlWebHdfsFileContextCreateMkdirLive.java
deleted file mode 100644
index fc50c04..0000000
--- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlWebHdfsFileContextCreateMkdirLive.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.hadoop.fs.adl.live;
-
-import org.apache.commons.lang.RandomStringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.DelegateToFileSystem;
-import org.apache.hadoop.fs.FileContext;
-import org.apache.hadoop.fs.FileContextCreateMkdirBaseTest;
-import org.apache.hadoop.fs.FileContextTestHelper;
-import org.apache.hadoop.fs.FileSystem;
-import org.junit.Assume;
-import org.junit.BeforeClass;
-
-import java.io.File;
-import java.net.URI;
-
-/**
- * Verify Adls file system adhere to Hadoop file system contract using bunch of
- * available test in FileContextCreateMkdirBaseTest.
- */
-public class TestAdlWebHdfsFileContextCreateMkdirLive
- extends FileContextCreateMkdirBaseTest {
- private static final String KEY_FILE_SYSTEM = "fs.defaultFS";
-
- @Override
- public void setUp() throws Exception {
- Configuration conf = AdlStorageConfiguration.getConfiguration();
- String fileSystem = conf.get(KEY_FILE_SYSTEM);
- if (fileSystem == null || fileSystem.trim().length() == 0) {
- throw new Exception("Default file system not configured.");
- }
- URI uri = new URI(fileSystem);
- FileSystem fs = AdlStorageConfiguration.createAdlStorageConnector();
- fc = FileContext.getFileContext(
- new DelegateToFileSystem(uri, fs, conf, fs.getScheme(), false) {
- }, conf);
- super.setUp();
- }
-
- /**
- * Required to override since the getRandmizedTestDir on Windows generates
- * absolute path of the local file path which contains ":" character.
- * Example file system path generated is "adl://<FileSystem Path>/d:/a/b/c
- *
- * Adls does not support : character in the path hence overriding to remove
- * unsupported character from the path.
- *
- * @return FileContextTestHelper
- */
- @Override
- protected FileContextTestHelper createFileContextHelper() {
- return new FileContextTestHelper(new File(
- RandomStringUtils.randomAlphanumeric(10))
- .getAbsolutePath().replaceAll(":", ""));
- }
-
- @BeforeClass
- public static void skipTestCheck() {
- Assume.assumeTrue(AdlStorageConfiguration.isContractTestEnabled());
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5c61ad24/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlWebHdfsFileContextMainOperationsLive.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlWebHdfsFileContextMainOperationsLive.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlWebHdfsFileContextMainOperationsLive.java
deleted file mode 100644
index b135550..0000000
--- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlWebHdfsFileContextMainOperationsLive.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.hadoop.fs.adl.live;
-
-import org.apache.commons.lang.RandomStringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.DelegateToFileSystem;
-import org.apache.hadoop.fs.FileContext;
-import org.apache.hadoop.fs.FileContextMainOperationsBaseTest;
-import org.apache.hadoop.fs.FileContextTestHelper;
-import org.apache.hadoop.fs.FileSystem;
-import org.junit.Assume;
-import org.junit.BeforeClass;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.URI;
-
-/**
- * Verify Adls file system adhere to Hadoop file system contract using bunch of
- * available test in FileContextMainOperationsBaseTest.
- */
-public class TestAdlWebHdfsFileContextMainOperationsLive
- extends FileContextMainOperationsBaseTest {
-
- private static final String KEY_FILE_SYSTEM = "fs.defaultFS";
-
- @BeforeClass
- public static void skipTestCheck() {
- Assume.assumeTrue(AdlStorageConfiguration.isContractTestEnabled());
- }
-
- @Override
- public void setUp() throws Exception {
- Configuration conf = AdlStorageConfiguration.getConfiguration();
- String fileSystem = conf.get(KEY_FILE_SYSTEM);
- if (fileSystem == null || fileSystem.trim().length() == 0) {
- throw new Exception("Default file system not configured.");
- }
- URI uri = new URI(fileSystem);
- FileSystem fs = AdlStorageConfiguration.createAdlStorageConnector();
- fc = FileContext.getFileContext(
- new DelegateToFileSystem(uri, fs, conf, fs.getScheme(), false) {
- }, conf);
- super.setUp();
- }
-
- /**
- * Required to override since the getRandmizedTestDir on Windows generates
- * absolute path of the local file path which contains ":" character.
- * Example file system path generated is "adl://<FileSystem Path>/d:/a/b/c
- *
- * Adls does not support : character in the path hence overriding to remove
- * unsupported character from the path.
- *
- * @return FileContextTestHelper
- */
- @Override
- protected FileContextTestHelper createFileContextHelper() {
- return new FileContextTestHelper(
- new File(RandomStringUtils.randomAlphanumeric(10)).getAbsolutePath()
- .replaceAll(":", ""));
- }
-
- @Override
- protected boolean listCorruptedBlocksSupported() {
- return false;
- }
-
- @Override
- public void testUnsupportedSymlink() throws IOException {
- Assume.assumeTrue("Symbolic link are not supported by Adls", false);
- }
-
- /**
- * In case this test is causing failure due to
- * java.lang.RuntimeException: java.io.FileNotFoundException: Hadoop bin
- * directory does not exist: <path>\hadoop-common-project
- * \hadoop-common\target\bin -see https://wiki.apache
- * .org/hadoop/WindowsProblems. then do build the hadoop dependencies
- * otherwise mark this test as skip.
- */
- @Override
- public void testWorkingDirectory() throws Exception {
- super.testWorkingDirectory();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5c61ad24/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/oauth2/TestCachedRefreshTokenBasedAccessTokenProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/oauth2/TestCachedRefreshTokenBasedAccessTokenProvider.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/oauth2/TestCachedRefreshTokenBasedAccessTokenProvider.java
deleted file mode 100644
index c044594..0000000
--- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/oauth2/TestCachedRefreshTokenBasedAccessTokenProvider.java
+++ /dev/null
@@ -1,149 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-package org.apache.hadoop.fs.adl.oauth2;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.web.oauth2.AccessTokenProvider;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TestName;
-
-import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.OAUTH_CLIENT_ID_KEY;
-import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.OAUTH_REFRESH_URL_KEY;
-import static org.apache.hadoop.hdfs.web.oauth2.ConfRefreshTokenBasedAccessTokenProvider.OAUTH_REFRESH_TOKEN_KEY;
-import static org.junit.Assert.fail;
-import static org.mockito.Mockito.any;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
-/**
- * Verify cache behavior of ConfRefreshTokenBasedAccessTokenProvider instances.
- */
-public class TestCachedRefreshTokenBasedAccessTokenProvider {
-
- private Configuration conf;
-
- @Rule public TestName name = new TestName();
- String clientId(int id) {
- return name.getMethodName() + "_clientID" + id;
- }
-
- @Before
- public void initConfig() {
- conf = new Configuration(false);
- conf.set(OAUTH_CLIENT_ID_KEY, clientId(0));
- conf.set(OAUTH_REFRESH_TOKEN_KEY, "01234567890abcdef");
- conf.set(OAUTH_REFRESH_URL_KEY, "http://dingo.invalid:80");
- }
-
- @Test
- public void testCacheInstance() throws Exception {
- final AccessTokenProvider inst0 = mock(AccessTokenProvider.class);
- when(inst0.getConf()).thenReturn(conf);
-
- // verify config
- CachedRefreshTokenBasedAccessTokenProvider t1 = new MockProvider(inst0);
- t1.setConf(conf);
- verify(inst0).setConf(any(Configuration.class)); // cloned, not exact match
-
- // verify cache hit
- CachedRefreshTokenBasedAccessTokenProvider t2 =
- new CachedRefreshTokenBasedAccessTokenProvider() {
- @Override
- AccessTokenProvider newInstance() {
- fail("Failed to return cached instance");
- return null;
- }
- };
- t2.setConf(conf);
-
- // verify force refresh
- conf.setBoolean(
- CachedRefreshTokenBasedAccessTokenProvider.FORCE_REFRESH, true);
- final AccessTokenProvider inst1 = mock(AccessTokenProvider.class);
- when(inst1.getConf()).thenReturn(conf);
- CachedRefreshTokenBasedAccessTokenProvider t3 = new MockProvider(inst1);
- t3.setConf(conf);
- verify(inst1).setConf(any(Configuration.class));
-
- // verify cache miss
- conf.set(OAUTH_REFRESH_URL_KEY, "http://yak.invalid:80");
- final AccessTokenProvider inst2 = mock(AccessTokenProvider.class);
- when(inst2.getConf()).thenReturn(conf);
- CachedRefreshTokenBasedAccessTokenProvider t4 = new MockProvider(inst2);
- t4.setConf(conf);
- verify(inst2).setConf(any(Configuration.class));
- }
-
- @Test
- public void testCacheLimit() throws Exception {
- final int iter = CachedRefreshTokenBasedAccessTokenProvider.MAX_PROVIDERS;
- for (int i = 0; i < iter; ++i) {
- conf.set(OAUTH_CLIENT_ID_KEY, clientId(i));
- AccessTokenProvider inst = mock(AccessTokenProvider.class);
- when(inst.getConf()).thenReturn(conf);
- CachedRefreshTokenBasedAccessTokenProvider t = new MockProvider(inst);
- t.setConf(conf);
- verify(inst).setConf(any(Configuration.class));
- }
- // verify cache hit
- for (int i = 0; i < iter; ++i) {
- conf.set(OAUTH_CLIENT_ID_KEY, clientId(i));
- CachedRefreshTokenBasedAccessTokenProvider t =
- new CachedRefreshTokenBasedAccessTokenProvider() {
- @Override
- AccessTokenProvider newInstance() {
- fail("Failed to return cached instance");
- return null;
- }
- };
- t.setConf(conf);
- }
-
- // verify miss, evict 0
- conf.set(OAUTH_CLIENT_ID_KEY, clientId(iter));
- final AccessTokenProvider inst = mock(AccessTokenProvider.class);
- when(inst.getConf()).thenReturn(conf);
- CachedRefreshTokenBasedAccessTokenProvider t = new MockProvider(inst);
- t.setConf(conf);
- verify(inst).setConf(any(Configuration.class));
-
- // verify miss
- conf.set(OAUTH_CLIENT_ID_KEY, clientId(0));
- final AccessTokenProvider inst0 = mock(AccessTokenProvider.class);
- when(inst0.getConf()).thenReturn(conf);
- CachedRefreshTokenBasedAccessTokenProvider t0 = new MockProvider(inst0);
- t0.setConf(conf);
- verify(inst0).setConf(any(Configuration.class));
- }
-
- static class MockProvider extends CachedRefreshTokenBasedAccessTokenProvider {
- private final AccessTokenProvider inst;
- MockProvider(AccessTokenProvider inst) {
- this.inst = inst;
- }
- @Override
- AccessTokenProvider newInstance() {
- return inst;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5c61ad24/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/common/AdlMockWebServer.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/common/AdlMockWebServer.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/common/AdlMockWebServer.java
deleted file mode 100644
index 69bb9ae..0000000
--- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/common/AdlMockWebServer.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.hadoop.fs.common;
-
-import com.eclipsesource.json.JsonObject;
-import com.squareup.okhttp.mockwebserver.MockResponse;
-import com.squareup.okhttp.mockwebserver.MockWebServer;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.adl.TestableAdlFileSystem;
-import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
-import org.apache.hadoop.hdfs.web.oauth2.ConfCredentialBasedAccessTokenProvider;
-import org.apache.hadoop.hdfs.web.oauth2.CredentialBasedAccessTokenProvider;
-import org.apache.hadoop.hdfs.web.oauth2.OAuth2Constants;
-import org.junit.After;
-import org.junit.Before;
-
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.URL;
-
-/**
- * Mock server to simulate Adls backend calls. This infrastructure is expandable
- * to override expected server response based on the derived test functionality.
- * Common functionality to generate token information before request is send to
- * adls backend is also managed within AdlMockWebServer implementation.
- */
-public class AdlMockWebServer {
-
- // Create a MockWebServer. These are lean enough that you can create a new
- // instance for every unit test.
- private MockWebServer server = null;
- private TestableAdlFileSystem fs = null;
- private int port = 0;
- private Configuration conf = new Configuration();
-
- public MockWebServer getMockServer() {
- return server;
- }
-
- public TestableAdlFileSystem getMockAdlFileSystem() {
- return fs;
- }
-
- public int getPort() {
- return port;
- }
-
- public Configuration getConf() {
- return conf;
- }
-
- public static MockResponse getTokenResponse() {
- JsonObject jsonObject = new JsonObject()
- .set(OAuth2Constants.EXPIRES_IN, "0987654321")
- .set("token_type", "bearer").set(OAuth2Constants.ACCESS_TOKEN, "123");
- MockResponse oauth2Response = new MockResponse();
- oauth2Response.addHeader("Content-Type", "application/json");
- oauth2Response.setResponseCode(200);
- oauth2Response.setBody(jsonObject.toString());
- return oauth2Response;
- }
-
- @Before
- public void preTestSetup() throws IOException, URISyntaxException {
-
- server = new MockWebServer();
- server.enqueue(getTokenResponse());
-
- // Start the server.
- server.start();
-
- // Ask the server for its URL. You'll need this to make HTTP requests.
- URL baseUrl = server.getUrl("");
- port = baseUrl.getPort();
-
- // Exercise your application code, which should make those HTTP requests.
- // Responses are returned in the same order that they are enqueued.
- fs = new TestableAdlFileSystem();
-
- conf.set(HdfsClientConfigKeys.OAUTH_CLIENT_ID_KEY, "MY_CLIENTID");
- conf.set(HdfsClientConfigKeys.ACCESS_TOKEN_PROVIDER_KEY,
- ConfCredentialBasedAccessTokenProvider.class.getName());
- conf.set(HdfsClientConfigKeys.DFS_WEBHDFS_OAUTH_ENABLED_KEY, "true");
- conf.set(HdfsClientConfigKeys.OAUTH_REFRESH_URL_KEY, "http://localhost:" +
- port + "/refresh");
- conf.set(CredentialBasedAccessTokenProvider.OAUTH_CREDENTIAL_KEY,
- "credential");
-
- URI uri = new URI("adl://localhost:" + port);
- fs.initialize(uri, conf);
- }
-
- @After
- public void postTestSetup() throws IOException {
- fs.close();
- server.shutdown();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5c61ad24/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/common/ExpectedResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/common/ExpectedResponse.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/common/ExpectedResponse.java
deleted file mode 100644
index 727417e..0000000
--- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/common/ExpectedResponse.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.hadoop.fs.common;
-
-
-import com.squareup.okhttp.mockwebserver.MockResponse;
-
-import java.util.ArrayList;
-
-/**
- * Supporting class to hold expected MockResponse object along with parameters
- * for validation in test methods.
- */
-public class ExpectedResponse {
- private MockResponse response;
- private ArrayList<String> expectedQueryParameters = new ArrayList<String>();
- private int expectedBodySize;
- private String httpRequestType;
-
- public int getExpectedBodySize() {
- return expectedBodySize;
- }
-
- public String getHttpRequestType() {
- return httpRequestType;
- }
-
- public ArrayList<String> getExpectedQueryParameters() {
- return expectedQueryParameters;
- }
-
- public MockResponse getResponse() {
- return response;
- }
-
- ExpectedResponse set(MockResponse mockResponse) {
- this.response = mockResponse;
- return this;
- }
-
- ExpectedResponse addExpectedQueryParam(String param) {
- expectedQueryParameters.add(param);
- return this;
- }
-
- ExpectedResponse addExpectedBodySize(int bodySize) {
- this.expectedBodySize = bodySize;
- return this;
- }
-
- ExpectedResponse addExpectedHttpRequestType(String expectedHttpRequestType) {
- this.httpRequestType = expectedHttpRequestType;
- return this;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5c61ad24/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/common/TestDataForRead.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/common/TestDataForRead.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/common/TestDataForRead.java
deleted file mode 100644
index 150dc6c..0000000
--- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/common/TestDataForRead.java
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.hadoop.fs.common;
-
-import com.squareup.okhttp.mockwebserver.Dispatcher;
-import com.squareup.okhttp.mockwebserver.MockResponse;
-import com.squareup.okhttp.mockwebserver.RecordedRequest;
-import okio.Buffer;
-import org.apache.hadoop.fs.adl.TestADLResponseData;
-
-import java.util.ArrayList;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-/**
- * Supporting class for mock test to validate Adls read operation using
- * BufferManager.java and BatchByteArrayInputStream implementation.
- */
-public class TestDataForRead {
-
- private byte[] actualData;
- private ArrayList<ExpectedResponse> responses;
- private Dispatcher dispatcher;
- private int intensityOfTest;
- private boolean checkOfNoOfCalls;
- private int expectedNoNetworkCall;
-
- public TestDataForRead(final byte[] actualData, int expectedNoNetworkCall,
- int intensityOfTest, boolean checkOfNoOfCalls) {
- this.checkOfNoOfCalls = checkOfNoOfCalls;
- this.actualData = actualData;
- responses = new ArrayList<ExpectedResponse>();
- this.expectedNoNetworkCall = expectedNoNetworkCall;
- this.intensityOfTest = intensityOfTest;
-
- dispatcher = new Dispatcher() {
- @Override
- public MockResponse dispatch(RecordedRequest recordedRequest)
- throws InterruptedException {
- if (recordedRequest.getPath().equals("/refresh")) {
- return AdlMockWebServer.getTokenResponse();
- }
-
- if (recordedRequest.getRequestLine().contains("op=GETFILESTATUS")) {
- return new MockResponse().setResponseCode(200).setBody(
- TestADLResponseData
- .getGetFileStatusJSONResponse(actualData.length));
- }
-
- if (recordedRequest.getRequestLine().contains("op=OPEN")) {
- String request = recordedRequest.getRequestLine();
- int offset = 0;
- int byteCount = 0;
-
- Pattern pattern = Pattern.compile("offset=([0-9]+)");
- Matcher matcher = pattern.matcher(request);
- if (matcher.find()) {
- System.out.println(matcher.group(1));
- offset = Integer.parseInt(matcher.group(1));
- }
-
- pattern = Pattern.compile("length=([0-9]+)");
- matcher = pattern.matcher(request);
- if (matcher.find()) {
- System.out.println(matcher.group(1));
- byteCount = Integer.parseInt(matcher.group(1));
- }
-
- Buffer buf = new Buffer();
- buf.write(actualData, offset, byteCount);
- return new MockResponse().setResponseCode(200)
- .setChunkedBody(buf, 4 * 1024 * 1024);
- }
-
- return new MockResponse().setBody("NOT SUPPORTED").setResponseCode(501);
- }
- };
- }
-
- public boolean isCheckOfNoOfCalls() {
- return checkOfNoOfCalls;
- }
-
- public int getExpectedNoNetworkCall() {
- return expectedNoNetworkCall;
- }
-
- public int getIntensityOfTest() {
- return intensityOfTest;
- }
-
- public byte[] getActualData() {
- return actualData;
- }
-
- public ArrayList<ExpectedResponse> getResponses() {
- return responses;
- }
-
- public Dispatcher getDispatcher() {
- return dispatcher;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5c61ad24/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/hdfs/web/TestAdlRead.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/hdfs/web/TestAdlRead.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/hdfs/web/TestAdlRead.java
deleted file mode 100644
index 60904d0..0000000
--- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/hdfs/web/TestAdlRead.java
+++ /dev/null
@@ -1,205 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.hadoop.hdfs.web;
-
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.adl.TestADLResponseData;
-import org.apache.hadoop.fs.common.AdlMockWebServer;
-import org.apache.hadoop.fs.common.TestDataForRead;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.Timeout;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Random;
-
-/**
- * This class is responsible for stress positional reads vs number of network
- * calls required by to fetch the amount of data. Test does ensure the data
- * integrity and order of the data is maintained. This tests are meant to test
- * BufferManager.java and BatchByteArrayInputStream implementation.
- */
-@RunWith(Parameterized.class)
-public class TestAdlRead extends AdlMockWebServer {
-
- // Keeping timeout of 1 hour to ensure the test does complete and should
- // not terminate due to high backend latency.
- @Rule
- public Timeout globalTimeout = new Timeout(60 * 60000);
- private TestDataForRead testData;
-
- public TestAdlRead(TestDataForRead testData) {
- this.testData = testData;
- getConf().set("adl.feature.override.readahead.max.buffersize", "8192");
- getConf().set("adl.feature.override.readahead.max.concurrent.connection",
- "1");
- }
-
- @Parameterized.Parameters(name = "{index}")
- public static Collection testDataForReadOperation() {
- return Arrays.asList(new Object[][] {
-
- //--------------------------
- // Test Data
- //--------------------------
- {new TestDataForRead("Hello World".getBytes(), 3, 1000, true)},
- {new TestDataForRead(
- ("the problem you appear to be wrestling with is that this doesn't "
- + "display very well. ").getBytes(), 3, 1000, true)},
- {new TestDataForRead(
- ("Chinese Indonesians (Indonesian: Orang Tionghoa-Indonesia; "
- + "Chinese: "
- + "trad ???????, simp ???????, pin Y\ufffdnd\ufffdn\ufffdx?y\ufffd Hu\ufffdr\ufffdn), are "
- + "Indonesians descended from various Chinese ethnic groups, "
- + "particularly Han.").getBytes(), 3, 1000, true)},
- {new TestDataForRead(
- TestADLResponseData.getRandomByteArrayData(5 * 1024), 3, 1000,
- true)}, {new TestDataForRead(
- TestADLResponseData.getRandomByteArrayData(1 * 1024), 3, 50, true)},
- {new TestDataForRead(
- TestADLResponseData.getRandomByteArrayData(8 * 1024), 3, 10, true)},
- {new TestDataForRead(
- TestADLResponseData.getRandomByteArrayData(32 * 1024), 6, 10,
- false)}, {new TestDataForRead(
- TestADLResponseData.getRandomByteArrayData(48 * 1024), 8, 10, false)}});
- }
-
- @After
- @Before
- public void cleanReadBuffer() {
- BufferManager.getInstance().clear();
- }
-
- @Test
- public void testEntireBytes() throws IOException, InterruptedException {
- getMockServer().setDispatcher(testData.getDispatcher());
- FSDataInputStream in = getMockAdlFileSystem().open(new Path("/test"));
- byte[] expectedData = new byte[testData.getActualData().length];
- Assert.assertEquals(in.read(expectedData), expectedData.length);
- Assert.assertArrayEquals(expectedData, testData.getActualData());
- in.close();
- if (testData.isCheckOfNoOfCalls()) {
- Assert.assertEquals(testData.getExpectedNoNetworkCall(),
- getMockServer().getRequestCount());
- }
- }
-
- @Test
- public void testSeekOperation() throws IOException, InterruptedException {
- getMockServer().setDispatcher(testData.getDispatcher());
- FSDataInputStream in = getMockAdlFileSystem().open(new Path("/test"));
- Random random = new Random();
- for (int i = 0; i < 1000; ++i) {
- int position = random.nextInt(testData.getActualData().length);
- in.seek(position);
- Assert.assertEquals(in.getPos(), position);
- Assert.assertEquals(in.read(), testData.getActualData()[position] & 0xFF);
- }
- in.close();
- if (testData.isCheckOfNoOfCalls()) {
- Assert.assertEquals(testData.getExpectedNoNetworkCall(),
- getMockServer().getRequestCount());
- }
- }
-
- @Test
- public void testReadServerCalls() throws IOException, InterruptedException {
- getMockServer().setDispatcher(testData.getDispatcher());
- FSDataInputStream in = getMockAdlFileSystem().open(new Path("/test"));
- byte[] expectedData = new byte[testData.getActualData().length];
- in.readFully(expectedData);
- Assert.assertArrayEquals(expectedData, testData.getActualData());
- Assert.assertEquals(testData.getExpectedNoNetworkCall(),
- getMockServer().getRequestCount());
- in.close();
- }
-
- @Test
- public void testReadFully() throws IOException, InterruptedException {
- getMockServer().setDispatcher(testData.getDispatcher());
- FSDataInputStream in = getMockAdlFileSystem().open(new Path("/test"));
- byte[] expectedData = new byte[testData.getActualData().length];
- in.readFully(expectedData);
- Assert.assertArrayEquals(expectedData, testData.getActualData());
-
- in.readFully(0, expectedData);
- Assert.assertArrayEquals(expectedData, testData.getActualData());
-
- in.seek(0);
- in.readFully(expectedData, 0, expectedData.length);
- Assert.assertArrayEquals(expectedData, testData.getActualData());
- in.close();
-
- if (testData.isCheckOfNoOfCalls()) {
- Assert.assertEquals(testData.getExpectedNoNetworkCall(),
- getMockServer().getRequestCount());
- }
- }
-
- @Test
- public void testRandomPositionalReadUsingReadFully()
- throws IOException, InterruptedException {
- getMockServer().setDispatcher(testData.getDispatcher());
- FSDataInputStream in = getMockAdlFileSystem().open(new Path("/test"));
- ByteArrayInputStream actualData = new ByteArrayInputStream(
- testData.getActualData());
- Random random = new Random();
- for (int i = 0; i < testData.getIntensityOfTest(); ++i) {
- int offset = random.nextInt(testData.getActualData().length);
- int length = testData.getActualData().length - offset;
- byte[] expectedData = new byte[length];
- byte[] actualDataSubset = new byte[length];
- actualData.reset();
- actualData.skip(offset);
- actualData.read(actualDataSubset, 0, length);
-
- in.readFully(offset, expectedData, 0, length);
- Assert.assertArrayEquals(expectedData, actualDataSubset);
- }
-
- for (int i = 0; i < testData.getIntensityOfTest(); ++i) {
- int offset = random.nextInt(testData.getActualData().length);
- int length = random.nextInt(testData.getActualData().length - offset);
- byte[] expectedData = new byte[length];
- byte[] actualDataSubset = new byte[length];
- actualData.reset();
- actualData.skip(offset);
- actualData.read(actualDataSubset, 0, length);
-
- in.readFully(offset, expectedData, 0, length);
- Assert.assertArrayEquals(expectedData, actualDataSubset);
- }
-
- in.close();
- if (testData.isCheckOfNoOfCalls()) {
- Assert.assertEquals(testData.getExpectedNoNetworkCall(),
- getMockServer().getRequestCount());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5c61ad24/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/hdfs/web/TestConcurrentDataReadOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/hdfs/web/TestConcurrentDataReadOperations.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/hdfs/web/TestConcurrentDataReadOperations.java
deleted file mode 100644
index 1194336..0000000
--- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/hdfs/web/TestConcurrentDataReadOperations.java
+++ /dev/null
@@ -1,306 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.hadoop.hdfs.web;
-
-import com.squareup.okhttp.mockwebserver.Dispatcher;
-import com.squareup.okhttp.mockwebserver.MockResponse;
-import com.squareup.okhttp.mockwebserver.RecordedRequest;
-import okio.Buffer;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.adl.TestADLResponseData;
-import org.apache.hadoop.fs.common.AdlMockWebServer;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Random;
-import java.util.UUID;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-/**
- * This class is responsible for testing multiple threads trying to access same
- * or multiple files from the offset. This tests are meant to test
- * BufferManager.java and BatchByteArrayInputStream implementation.
- */
-@RunWith(Parameterized.class)
-public class TestConcurrentDataReadOperations extends AdlMockWebServer {
-
- private static FSDataInputStream commonHandle = null;
- private static Object lock = new Object();
- private int concurrencyLevel;
-
- public TestConcurrentDataReadOperations(int concurrencyLevel) {
- this.concurrencyLevel = concurrencyLevel;
- getConf().set("adl.feature.override.readahead.max.buffersize", "102400");
- getConf().set("adl.feature.override.readahead.max.concurrent.connection",
- "1");
- }
-
- @Parameterized.Parameters(name = "{index}")
- public static Collection testDataNumberOfConcurrentRun() {
- return Arrays.asList(new Object[][] {{1}, {2}, {3}, {4}, {5}});
- }
-
- public static byte[] getRandomByteArrayData(int size) {
- byte[] b = new byte[size];
- Random rand = new Random();
- rand.nextBytes(b);
- return b;
- }
-
- private void setDispatcher(final ArrayList<CreateTestData> testData) {
- getMockServer().setDispatcher(new Dispatcher() {
- @Override
- public MockResponse dispatch(RecordedRequest recordedRequest)
- throws InterruptedException {
- if (recordedRequest.getPath().equals("/refresh")) {
- return AdlMockWebServer.getTokenResponse();
- }
-
- CreateTestData currentRequest = null;
- for (CreateTestData local : testData) {
- if (recordedRequest.getPath().contains(local.path.toString())) {
- currentRequest = local;
- break;
- }
- }
-
- if (currentRequest == null) {
- new MockResponse().setBody("Request data not found")
- .setResponseCode(501);
- }
-
- if (recordedRequest.getRequestLine().contains("op=GETFILESTATUS")) {
- return new MockResponse().setResponseCode(200).setBody(
- TestADLResponseData
- .getGetFileStatusJSONResponse(currentRequest.data.length));
- }
-
- if (recordedRequest.getRequestLine().contains("op=OPEN")) {
- String request = recordedRequest.getRequestLine();
- int offset = 0;
- int byteCount = 0;
-
- Pattern pattern = Pattern.compile("offset=([0-9]+)");
- Matcher matcher = pattern.matcher(request);
- if (matcher.find()) {
- System.out.println(matcher.group(1));
- offset = Integer.parseInt(matcher.group(1));
- }
-
- pattern = Pattern.compile("length=([0-9]+)");
- matcher = pattern.matcher(request);
- if (matcher.find()) {
- System.out.println(matcher.group(1));
- byteCount = Integer.parseInt(matcher.group(1));
- }
-
- Buffer buf = new Buffer();
- buf.write(currentRequest.data, offset, byteCount);
- return new MockResponse().setResponseCode(200)
- .setChunkedBody(buf, 4 * 1024 * 1024);
- }
-
- return new MockResponse().setBody("NOT SUPPORTED").setResponseCode(501);
- }
- });
- }
-
- @Before
- public void resetHandle() {
- commonHandle = null;
- }
-
- @Test
- public void testParallelReadOnDifferentStreams()
- throws IOException, InterruptedException, ExecutionException {
-
- ArrayList<CreateTestData> createTestData = new ArrayList<CreateTestData>();
-
- Random random = new Random();
-
- for (int i = 0; i < concurrencyLevel; i++) {
- CreateTestData testData = new CreateTestData();
- testData
- .set(new Path("/test/concurrentRead/" + UUID.randomUUID().toString()),
- getRandomByteArrayData(random.nextInt(1 * 1024 * 1024)));
- createTestData.add(testData);
- }
-
- setDispatcher(createTestData);
-
- ArrayList<ReadTestData> readTestData = new ArrayList<ReadTestData>();
- for (CreateTestData local : createTestData) {
- ReadTestData localReadData = new ReadTestData();
- localReadData.set(local.path, local.data, 0);
- readTestData.add(localReadData);
- }
-
- runReadTest(readTestData, false);
- }
-
- @Test
- public void testParallelReadOnSameStreams()
- throws IOException, InterruptedException, ExecutionException {
- ArrayList<CreateTestData> createTestData = new ArrayList<CreateTestData>();
-
- Random random = new Random();
-
- for (int i = 0; i < 1; i++) {
- CreateTestData testData = new CreateTestData();
- testData
- .set(new Path("/test/concurrentRead/" + UUID.randomUUID().toString()),
- getRandomByteArrayData(1024 * 1024));
- createTestData.add(testData);
- }
-
- setDispatcher(createTestData);
-
- ArrayList<ReadTestData> readTestData = new ArrayList<ReadTestData>();
- ByteArrayInputStream buffered = new ByteArrayInputStream(
- createTestData.get(0).data);
-
- ReadTestData readInitially = new ReadTestData();
- byte[] initialData = new byte[1024 * 1024];
- buffered.read(initialData);
-
- readInitially.set(createTestData.get(0).path, initialData, 0);
- readTestData.add(readInitially);
- runReadTest(readTestData, false);
-
- readTestData.clear();
-
- for (int i = 0; i < concurrencyLevel * 5; i++) {
- ReadTestData localReadData = new ReadTestData();
- int offset = random.nextInt((1024 * 1024)-1);
- int length = 1024 * 1024 - offset;
- byte[] expectedData = new byte[length];
- buffered.reset();
- buffered.skip(offset);
- buffered.read(expectedData);
- localReadData.set(createTestData.get(0).path, expectedData, offset);
- readTestData.add(localReadData);
- }
-
- runReadTest(readTestData, true);
- }
-
- void runReadTest(ArrayList<ReadTestData> testData, boolean useSameStream)
- throws InterruptedException, ExecutionException {
-
- ExecutorService executor = Executors.newFixedThreadPool(testData.size());
- Future[] subtasks = new Future[testData.size()];
-
- for (int i = 0; i < testData.size(); i++) {
- subtasks[i] = executor.submit(
- new ReadConcurrentRunnable(testData.get(i).data, testData.get(i).path,
- testData.get(i).offset, useSameStream));
- }
-
- executor.shutdown();
-
- // wait until all tasks are finished
- executor.awaitTermination(120, TimeUnit.SECONDS);
-
- for (int i = 0; i < testData.size(); ++i) {
- Assert.assertTrue((Boolean) subtasks[i].get());
- }
- }
-
- class ReadTestData {
- private Path path;
- private byte[] data;
- private int offset;
-
- public void set(Path filePath, byte[] dataToBeRead, int fromOffset) {
- this.path = filePath;
- this.data = dataToBeRead;
- this.offset = fromOffset;
- }
- }
-
- class CreateTestData {
- private Path path;
- private byte[] data;
-
- public void set(Path filePath, byte[] dataToBeWritten) {
- this.path = filePath;
- this.data = dataToBeWritten;
- }
- }
-
- class ReadConcurrentRunnable implements Callable<Boolean> {
- private Path path;
- private int offset;
- private byte[] expectedData;
- private boolean useSameStream;
-
- public ReadConcurrentRunnable(byte[] expectedData, Path path, int offset,
- boolean useSameStream) {
- this.path = path;
- this.offset = offset;
- this.expectedData = expectedData;
- this.useSameStream = useSameStream;
- }
-
- public Boolean call() throws IOException {
- try {
- FSDataInputStream in;
- if (useSameStream) {
- synchronized (lock) {
- if (commonHandle == null) {
- commonHandle = getMockAdlFileSystem().open(path);
- }
- in = commonHandle;
- }
- } else {
- in = getMockAdlFileSystem().open(path);
- }
-
- byte[] actualData = new byte[expectedData.length];
- in.readFully(offset, actualData);
- Assert.assertArrayEquals("Path :" + path.toString() + " did not match.",
- expectedData, actualData);
- if (!useSameStream) {
- in.close();
- }
- } catch (IOException e) {
- e.printStackTrace();
- return false;
- }
- return true;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5c61ad24/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/hdfs/web/TestConfigurationSetting.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/hdfs/web/TestConfigurationSetting.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/hdfs/web/TestConfigurationSetting.java
deleted file mode 100644
index 9f33fb4..0000000
--- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/hdfs/web/TestConfigurationSetting.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.hadoop.hdfs.web;
-
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.adl.TestableAdlFileSystem;
-import org.apache.hadoop.hdfs.web.oauth2.AccessTokenProvider;
-import org.apache.hadoop.hdfs.web.oauth2.ConfCredentialBasedAccessTokenProvider;
-import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.ACCESS_TOKEN_PROVIDER_KEY;
-import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_WEBHDFS_OAUTH_ENABLED_KEY;
-import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.OAUTH_CLIENT_ID_KEY;
-import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.OAUTH_REFRESH_URL_KEY;
-import static org.apache.hadoop.hdfs.web.oauth2.CredentialBasedAccessTokenProvider.OAUTH_CREDENTIAL_KEY;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-/**
- * This class is responsible for testing adl file system required configuration
- * and feature set keys.
- */
-public class TestConfigurationSetting {
-
- @Test
- public void testAllConfiguration() throws URISyntaxException, IOException {
- TestableAdlFileSystem fs = new TestableAdlFileSystem();
- Configuration conf = new Configuration();
- conf.set(OAUTH_REFRESH_URL_KEY, "http://localhost:1111/refresh");
- conf.set(OAUTH_CREDENTIAL_KEY, "credential");
- conf.set(OAUTH_CLIENT_ID_KEY, "MY_CLIENTID");
- conf.setClass(ACCESS_TOKEN_PROVIDER_KEY,
- ConfCredentialBasedAccessTokenProvider.class,
- AccessTokenProvider.class);
- conf.setBoolean(DFS_WEBHDFS_OAUTH_ENABLED_KEY, true);
-
- URI uri = new URI("adl://localhost:1234");
- fs.initialize(uri, conf);
-
- // Default setting check
- Assert.assertEquals(true, fs.isFeatureRedirectOff());
- Assert.assertEquals(true, fs.isFeatureGetBlockLocationLocallyBundled());
- Assert.assertEquals(true, fs.isFeatureConcurrentReadWithReadAhead());
- Assert.assertEquals(false, fs.isOverrideOwnerFeatureOn());
- Assert.assertEquals(8 * 1024 * 1024, fs.getMaxBufferSize());
- Assert.assertEquals(2, fs.getMaxConcurrentConnection());
-
- fs.close();
-
- // Configuration toggle check
- conf.set("adl.feature.override.redirection.off", "false");
- fs.initialize(uri, conf);
- Assert.assertEquals(false, fs.isFeatureRedirectOff());
- fs.close();
- conf.set("adl.feature.override.redirection.off", "true");
- fs.initialize(uri, conf);
- Assert.assertEquals(true, fs.isFeatureRedirectOff());
- fs.close();
-
- conf.set("adl.feature.override.getblocklocation.locally.bundled", "false");
- fs.initialize(uri, conf);
- Assert.assertEquals(false, fs.isFeatureGetBlockLocationLocallyBundled());
- fs.close();
- conf.set("adl.feature.override.getblocklocation.locally.bundled", "true");
- fs.initialize(uri, conf);
- Assert.assertEquals(true, fs.isFeatureGetBlockLocationLocallyBundled());
- fs.close();
-
- conf.set("adl.feature.override.readahead", "false");
- fs.initialize(uri, conf);
- Assert.assertEquals(false, fs.isFeatureConcurrentReadWithReadAhead());
- fs.close();
- conf.set("adl.feature.override.readahead", "true");
- fs.initialize(uri, conf);
- Assert.assertEquals(true, fs.isFeatureConcurrentReadWithReadAhead());
- fs.close();
-
- conf.set("adl.feature.override.readahead.max.buffersize", "101");
- fs.initialize(uri, conf);
- Assert.assertEquals(101, fs.getMaxBufferSize());
- fs.close();
- conf.set("adl.feature.override.readahead.max.buffersize", "12134565");
- fs.initialize(uri, conf);
- Assert.assertEquals(12134565, fs.getMaxBufferSize());
- fs.close();
-
- conf.set("adl.debug.override.localuserasfileowner", "true");
- fs.initialize(uri, conf);
- Assert.assertEquals(true, fs.isOverrideOwnerFeatureOn());
- fs.close();
- conf.set("adl.debug.override.localuserasfileowner", "false");
- fs.initialize(uri, conf);
- Assert.assertEquals(false, fs.isOverrideOwnerFeatureOn());
- fs.close();
- }
-
- @Test
- public void testOAuthEnable() throws Exception {
- try (TestableAdlFileSystem fs = new TestableAdlFileSystem()) {
- Configuration conf = new Configuration();
- conf.set(OAUTH_REFRESH_URL_KEY, "http://localhost:1111/refresh");
- conf.set(OAUTH_CREDENTIAL_KEY, "credential");
- conf.set(OAUTH_CLIENT_ID_KEY, "MY_CLIENTID");
- conf.setClass(ACCESS_TOKEN_PROVIDER_KEY,
- ConfCredentialBasedAccessTokenProvider.class,
- AccessTokenProvider.class);
- // disable OAuth2 in configuration, verify overridden
- conf.setBoolean(DFS_WEBHDFS_OAUTH_ENABLED_KEY, false);
-
- URI uri = new URI("adl://localhost:1234");
- fs.initialize(uri, conf);
- Assert.assertFalse(conf.getBoolean(DFS_WEBHDFS_OAUTH_ENABLED_KEY, false));
- Assert.assertTrue(fs.getConf().getBoolean(DFS_WEBHDFS_OAUTH_ENABLED_KEY,
- false));
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5c61ad24/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/hdfs/web/TestSplitSizeCalculation.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/hdfs/web/TestSplitSizeCalculation.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/hdfs/web/TestSplitSizeCalculation.java
deleted file mode 100644
index 42a4701..0000000
--- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/hdfs/web/TestSplitSizeCalculation.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.hadoop.hdfs.web;
-
-import com.squareup.okhttp.mockwebserver.MockResponse;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.adl.TestADLResponseData;
-import org.apache.hadoop.fs.common.AdlMockWebServer;
-import org.apache.hadoop.hdfs.web.PrivateAzureDataLakeFileSystem.BatchByteArrayInputStream;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.net.URISyntaxException;
-import java.net.URL;
-
-/**
- * This class is responsible for testing split size calculation during
- * read ahead buffer initiation based on the data size and configuration
- * initialization.
- */
-public class TestSplitSizeCalculation extends AdlMockWebServer {
-
- @Test
- public void testSplitSizeCalculations()
- throws URISyntaxException, IOException {
-
- getMockServer().enqueue(new MockResponse().setResponseCode(200).setBody(
- TestADLResponseData.getGetFileStatusJSONResponse(128 * 1024 * 1024)));
- getMockServer().enqueue(new MockResponse().setResponseCode(200).setBody(
- TestADLResponseData.getGetFileStatusJSONResponse(128 * 1024 * 1024)));
- getMockServer().enqueue(new MockResponse().setResponseCode(200).setBody(
- TestADLResponseData.getGetFileStatusJSONResponse(128 * 1024 * 1024)));
- getMockServer().enqueue(new MockResponse().setResponseCode(200).setBody(
- TestADLResponseData.getGetFileStatusJSONResponse(128 * 1024 * 1024)));
-
- URL url = getMockServer().getUrl("");
-
- BatchByteArrayInputStream stream = getMockAdlFileSystem()
- .new BatchByteArrayInputStream(url,
- new Path("/test1/test2"), 16 * 1024 * 1024, 4);
- Assert.assertEquals(1, stream.getSplitSize(1 * 1024 * 1024));
- Assert.assertEquals(1, stream.getSplitSize(2 * 1024 * 1024));
- Assert.assertEquals(1, stream.getSplitSize(3 * 1024 * 1024));
- Assert.assertEquals(1, stream.getSplitSize(4 * 1024 * 1024));
- Assert.assertEquals(1, stream.getSplitSize(5 * 1024 * 1024));
- Assert.assertEquals(1, stream.getSplitSize(6 * 1024 * 1024));
- Assert.assertEquals(1, stream.getSplitSize(7 * 1024 * 1024));
- Assert.assertEquals(2, stream.getSplitSize(8 * 1024 * 1024));
- Assert.assertEquals(4, stream.getSplitSize(16 * 1024 * 1024));
- Assert.assertEquals(3, stream.getSplitSize(12 * 1024 * 1024));
- Assert.assertEquals(4, stream.getSplitSize(102 * 1024 * 1024));
- Assert.assertEquals(1, stream.getSplitSize(102));
- stream.close();
-
- stream = getMockAdlFileSystem().new BatchByteArrayInputStream(url,
- new Path("/test1/test2"), 4 * 1024 * 1024, 4);
- Assert.assertEquals(1, stream.getSplitSize(1 * 1024 * 1024));
- Assert.assertEquals(1, stream.getSplitSize(2 * 1024 * 1024));
- Assert.assertEquals(1, stream.getSplitSize(3 * 1024 * 1024));
- Assert.assertEquals(1, stream.getSplitSize(4 * 1024 * 1024));
- Assert.assertEquals(1, stream.getSplitSize(5 * 1024 * 1024));
- Assert.assertEquals(1, stream.getSplitSize(8 * 1024 * 1024));
- Assert.assertEquals(1, stream.getSplitSize(5 * 1024 * 1024));
- Assert.assertEquals(1, stream.getSplitSize(6 * 1024 * 1024));
- Assert.assertEquals(1, stream.getSplitSize(7 * 1024 * 1024));
- Assert.assertEquals(1, stream.getSplitSize(16 * 1024 * 1024));
- Assert.assertEquals(1, stream.getSplitSize(12 * 1024 * 1024));
- Assert.assertEquals(1, stream.getSplitSize(102 * 1024 * 1024));
- Assert.assertEquals(1, stream.getSplitSize(102));
- stream.close();
-
- stream = getMockAdlFileSystem().new BatchByteArrayInputStream(url,
- new Path("/test1/test2"), 16 * 1024 * 1024, 2);
- Assert.assertEquals(1, stream.getSplitSize(1 * 1024 * 1024));
- Assert.assertEquals(1, stream.getSplitSize(2 * 1024 * 1024));
- Assert.assertEquals(1, stream.getSplitSize(3 * 1024 * 1024));
- Assert.assertEquals(1, stream.getSplitSize(4 * 1024 * 1024));
- Assert.assertEquals(1, stream.getSplitSize(5 * 1024 * 1024));
- Assert.assertEquals(1, stream.getSplitSize(5 * 1024 * 1024));
- Assert.assertEquals(1, stream.getSplitSize(6 * 1024 * 1024));
- Assert.assertEquals(1, stream.getSplitSize(7 * 1024 * 1024));
- Assert.assertEquals(2, stream.getSplitSize(8 * 1024 * 1024));
- Assert.assertEquals(2, stream.getSplitSize(16 * 1024 * 1024));
- Assert.assertEquals(2, stream.getSplitSize(12 * 1024 * 1024));
- Assert.assertEquals(2, stream.getSplitSize(102 * 1024 * 1024));
- Assert.assertEquals(1, stream.getSplitSize(102));
- stream.close();
-
- stream = getMockAdlFileSystem().new BatchByteArrayInputStream(url,
- new Path("/test1/test2"), 8 * 1024 * 1024, 2);
- Assert.assertEquals(1, stream.getSplitSize(1 * 1024 * 1024));
- Assert.assertEquals(1, stream.getSplitSize(2 * 1024 * 1024));
- Assert.assertEquals(1, stream.getSplitSize(3 * 1024 * 1024));
- Assert.assertEquals(1, stream.getSplitSize(4 * 1024 * 1024));
- Assert.assertEquals(1, stream.getSplitSize(5 * 1024 * 1024));
- Assert.assertEquals(1, stream.getSplitSize(6 * 1024 * 1024));
- Assert.assertEquals(1, stream.getSplitSize(7 * 1024 * 1024));
- Assert.assertEquals(2, stream.getSplitSize(8 * 1024 * 1024));
- Assert.assertEquals(2, stream.getSplitSize(16 * 1024 * 1024));
- Assert.assertEquals(2, stream.getSplitSize(12 * 1024 * 1024));
- Assert.assertEquals(2, stream.getSplitSize(102 * 1024 * 1024));
- Assert.assertEquals(1, stream.getSplitSize(102));
- stream.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5c61ad24/hadoop-tools/hadoop-azure-datalake/src/test/resources/adls.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/resources/adls.xml b/hadoop-tools/hadoop-azure-datalake/src/test/resources/adls.xml
index f72de16..76a78b0 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/test/resources/adls.xml
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/resources/adls.xml
@@ -39,22 +39,22 @@
<property>
<name>fs.contract.rename-returns-true-if-dest-exists</name>
- <value>true</value>
+ <value>false</value>
</property>
<property>
<name>fs.contract.rename-returns-true-if-source-missing</name>
- <value>true</value>
+ <value>false</value>
</property>
<property>
<name>fs.contract.rename-creates-dest-dirs</name>
- <value>true</value>
+ <value>false</value>
</property>
<property>
<name>fs.contract.rename-remove-dest-if-empty-dir</name>
- <value>true</value>
+ <value>false</value>
</property>
<property>
@@ -119,7 +119,7 @@
<property>
<name>fs.contract.rename-overwrites-dest</name>
- <value>true</value>
+ <value>false</value>
</property>
<property>
@@ -136,4 +136,5 @@
<name>fs.contract.supports-getfilestatus</name>
<value>true</value>
</property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5c61ad24/hadoop-tools/hadoop-azure-datalake/src/test/resources/contract-test-options.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/resources/contract-test-options.xml b/hadoop-tools/hadoop-azure-datalake/src/test/resources/contract-test-options.xml
index 160fba4..646922e 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/test/resources/contract-test-options.xml
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/resources/contract-test-options.xml
@@ -13,45 +13,49 @@
-->
<configuration>
<property>
- <name>dfs.webhdfs.oauth2.refresh.token.expires.ms.since.epoch</name>
- <value>0</value>
- </property>
- <property>
- <name>dfs.webhdfs.oauth2.credential</name>
- <value>bearer.and.refresh.token</value>
- </property>
- <property>
- <name>dfs.webhdfs.oauth2.refresh.url</name>
- <value>https://login.windows.net/common/oauth2/token/</value>
- </property>
- <property>
- <name>dfs.webhdfs.oauth2.access.token.provider</name>
+ <name>dfs.adls.oauth2.refresh.url</name>
<value>
- org.apache.hadoop.fs.adl.oauth2.CachedRefreshTokenBasedAccessTokenProvider
</value>
</property>
+
<property>
- <name>dfs.webhdfs.oauth2.enabled</name>
- <value>true</value>
- </property>
- <!--USER INPUT REQUIRED-->
- <property>
- <name>dfs.webhdfs.oauth2.client.id</name>
- <value>ADD CLIENT ID</value>
+ <name>dfs.adls.oauth2.credential</name>
+ <value></value>
</property>
- <!--USER INPUT REQUIRED-->
+
<property>
- <name>dfs.webhdfs.oauth2.refresh.token</name>
- <value>ADD REFRESH TOKEN</value>
+ <name>dfs.adls.oauth2.client.id</name>
+ <value></value>
</property>
- <!--USER INPUT REQUIRED-->
+
<property>
- <name>fs.defaultFS</name>
- <value>adl://urAdlAccountGoesHere.azuredatalakestore.net:443/</value>
+ <name>dfs.adls.oauth2.access.token.provider.type</name>
+ <value>ClientCredential</value>
+ <description>
+ Supported provider type:
+
+ "ClientCredential" : Client id and client credentials(Provided
+ through configuration file) flow.
+
+ "RefreshToken" : Client id and refresh token(Provided
+ through configuration file)flow.
+
+ "Custom" : Custom AAD token management.
+ </description>
</property>
- <!--USER INPUT REQUIRED-->
+
<property>
<name>dfs.adl.test.contract.enable</name>
<value>false</value>
</property>
-</configuration>
\ No newline at end of file
+
+ <property>
+ <name>test.fs.adl.name</name>
+ <value></value>
+ </property>
+
+ <property>
+ <name>fs.contract.test.fs</name>
+ <value>org.apache.hadoop.fs.adl.AdlFileSystem</value>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5c61ad24/hadoop-tools/hadoop-azure-datalake/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/resources/log4j.properties b/hadoop-tools/hadoop-azure-datalake/src/test/resources/log4j.properties
new file mode 100644
index 0000000..4cc8f7f
--- /dev/null
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/resources/log4j.properties
@@ -0,0 +1,30 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+log4j.rootLogger=DEBUG,stdout
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d [%t] %-5p %X{file} %c{1} - %m%n
+log4j.logger.your.app=*
+log4j.additivity.your.app=false
+log4j.logger.yourApp=*
+log4j.additivity.yourApp=false
+log4j.appender.yourApp=org.apache.log4j.ConsoleAppender
+log4j.appender.yourApp.layout=org.apache.log4j.PatternLayout
+log4j.appender.yourApp.layout.ConversionPattern=%d [%t] %-5p %X{file} %c{1} %m%n
+log4j.appender.yourApp.ImmediateFlush=true
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org