You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2018/02/12 15:13:33 UTC
[2/2] hadoop git commit: HADOOP-15187. Remove ADL mock test
dependency on REST call invoked from Java SDK. Contributed by Vishwajeet
Dusane.
HADOOP-15187. Remove ADL mock test dependency on REST call invoked from Java SDK.
Contributed by Vishwajeet Dusane.
(cherry picked from commit bd5ab5912564d2d687651b01f552b8e4ca8c145a)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8cf88fcd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8cf88fcd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8cf88fcd
Branch: refs/heads/trunk
Commit: 8cf88fcd1f63d3d4e9736b1b687a4f4e663f6125
Parents: d02e42c
Author: Steve Loughran <st...@apache.org>
Authored: Mon Feb 12 15:13:00 2018 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Feb 12 15:13:00 2018 +0000
----------------------------------------------------------------------
.../apache/hadoop/fs/adl/AdlMockWebServer.java | 102 -------
.../apache/hadoop/fs/adl/TestACLFeatures.java | 262 ----------------
.../org/apache/hadoop/fs/adl/TestAdlRead.java | 196 ------------
.../adl/TestConcurrentDataReadOperations.java | 299 -------------------
.../hadoop/fs/adl/TestCustomTokenProvider.java | 140 ---------
.../apache/hadoop/fs/adl/TestGetFileStatus.java | 102 -------
.../apache/hadoop/fs/adl/TestListStatus.java | 137 ---------
.../hadoop/fs/adl/TestableAdlFileSystem.java | 30 --
.../hadoop/fs/adl/common/ExpectedResponse.java | 71 -----
.../hadoop/fs/adl/common/TestDataForRead.java | 122 --------
10 files changed, 1461 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cf88fcd/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/AdlMockWebServer.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/AdlMockWebServer.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/AdlMockWebServer.java
deleted file mode 100644
index d843d55..0000000
--- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/AdlMockWebServer.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.adl;
-
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.URL;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.adl.common.CustomMockTokenProvider;
-import org.apache.hadoop.fs.adl.oauth2.AzureADTokenProvider;
-import static org.apache.hadoop.fs.adl.AdlConfKeys
- .AZURE_AD_TOKEN_PROVIDER_CLASS_KEY;
-import static org.apache.hadoop.fs.adl.AdlConfKeys
- .AZURE_AD_TOKEN_PROVIDER_TYPE_KEY;
-
-import com.squareup.okhttp.mockwebserver.MockWebServer;
-
-import org.junit.After;
-import org.junit.Before;
-
-/**
- * Mock server to simulate Adls backend calls. This infrastructure is expandable
- * to override expected server response based on the derived test functionality.
- * Common functionality to generate token information before request is send to
- * adls backend is also managed within AdlMockWebServer implementation using
- * {@link org.apache.hadoop.fs.adl.common.CustomMockTokenProvider}.
- */
-public class AdlMockWebServer {
- // Create a MockWebServer. These are lean enough that you can create a new
- // instance for every unit test.
- private MockWebServer server = null;
- private TestableAdlFileSystem fs = null;
- private int port = 0;
- private Configuration conf = new Configuration();
-
- public MockWebServer getMockServer() {
- return server;
- }
-
- public TestableAdlFileSystem getMockAdlFileSystem() {
- return fs;
- }
-
- public int getPort() {
- return port;
- }
-
- public Configuration getConf() {
- return conf;
- }
-
- public void setConf(Configuration conf) {
- this.conf = conf;
- }
-
- @Before
- public void preTestSetup() throws IOException, URISyntaxException {
- server = new MockWebServer();
-
- // Start the server.
- server.start();
-
- // Ask the server for its URL. You'll need this to make HTTP requests.
- URL baseUrl = server.getUrl("");
- port = baseUrl.getPort();
-
- // Exercise your application code, which should make those HTTP requests.
- // Responses are returned in the same order that they are enqueued.
- fs = new TestableAdlFileSystem();
-
- conf.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, TokenProviderType.Custom);
- conf.setClass(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY,
- CustomMockTokenProvider.class, AzureADTokenProvider.class);
-
- URI uri = new URI("adl://localhost:" + port);
- fs.initialize(uri, conf);
- }
-
- @After
- public void postTestSetup() throws IOException {
- fs.close();
- server.shutdown();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cf88fcd/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestACLFeatures.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestACLFeatures.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestACLFeatures.java
deleted file mode 100644
index b420daa..0000000
--- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestACLFeatures.java
+++ /dev/null
@@ -1,262 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.hadoop.fs.adl;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.net.URISyntaxException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.AclEntry;
-import org.apache.hadoop.fs.permission.AclEntryScope;
-import org.apache.hadoop.fs.permission.AclEntryType;
-import org.apache.hadoop.fs.permission.AclStatus;
-import org.apache.hadoop.fs.permission.FsAction;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.security.AccessControlException;
-
-import com.squareup.okhttp.mockwebserver.MockResponse;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-/**
- * Stub adl server and test acl data conversion within SDK and Hadoop adl
- * client.
- */
-public class TestACLFeatures extends AdlMockWebServer {
-
- @Test(expected=AccessControlException.class)
- public void testModifyAclEntries() throws URISyntaxException, IOException {
- getMockServer().enqueue(new MockResponse().setResponseCode(200));
- List<AclEntry> entries = new ArrayList<AclEntry>();
- AclEntry.Builder aclEntryBuilder = new AclEntry.Builder();
- aclEntryBuilder.setName("hadoop");
- aclEntryBuilder.setType(AclEntryType.USER);
- aclEntryBuilder.setPermission(FsAction.ALL);
- aclEntryBuilder.setScope(AclEntryScope.ACCESS);
- entries.add(aclEntryBuilder.build());
-
- aclEntryBuilder.setName("hdfs");
- aclEntryBuilder.setType(AclEntryType.GROUP);
- aclEntryBuilder.setPermission(FsAction.READ_WRITE);
- aclEntryBuilder.setScope(AclEntryScope.DEFAULT);
- entries.add(aclEntryBuilder.build());
-
- getMockAdlFileSystem().modifyAclEntries(new Path("/test1/test2"), entries);
-
- getMockServer().enqueue(new MockResponse().setResponseCode(403)
- .setBody(TestADLResponseData.getAccessControlException()));
-
- getMockAdlFileSystem()
- .modifyAclEntries(new Path("/test1/test2"), entries);
- }
-
- @Test(expected=AccessControlException.class)
- public void testRemoveAclEntriesWithOnlyUsers()
- throws URISyntaxException, IOException {
- getMockServer().enqueue(new MockResponse().setResponseCode(200));
- List<AclEntry> entries = new ArrayList<AclEntry>();
- AclEntry.Builder aclEntryBuilder = new AclEntry.Builder();
- aclEntryBuilder.setName("hadoop");
- aclEntryBuilder.setType(AclEntryType.USER);
- entries.add(aclEntryBuilder.build());
-
- getMockAdlFileSystem().removeAclEntries(new Path("/test1/test2"), entries);
-
- getMockServer().enqueue(new MockResponse().setResponseCode(403)
- .setBody(TestADLResponseData.getAccessControlException()));
-
- getMockAdlFileSystem()
- .removeAclEntries(new Path("/test1/test2"), entries);
- }
-
- @Test(expected=AccessControlException.class)
- public void testRemoveAclEntries() throws URISyntaxException, IOException {
- getMockServer().enqueue(new MockResponse().setResponseCode(200));
- List<AclEntry> entries = new ArrayList<AclEntry>();
- AclEntry.Builder aclEntryBuilder = new AclEntry.Builder();
- aclEntryBuilder.setName("hadoop");
- aclEntryBuilder.setType(AclEntryType.USER);
- aclEntryBuilder.setPermission(FsAction.ALL);
- aclEntryBuilder.setScope(AclEntryScope.ACCESS);
- entries.add(aclEntryBuilder.build());
-
- aclEntryBuilder.setName("hdfs");
- aclEntryBuilder.setType(AclEntryType.GROUP);
- aclEntryBuilder.setPermission(FsAction.READ_WRITE);
- aclEntryBuilder.setScope(AclEntryScope.DEFAULT);
- entries.add(aclEntryBuilder.build());
-
- getMockAdlFileSystem().removeAclEntries(new Path("/test1/test2"), entries);
-
- getMockServer().enqueue(new MockResponse().setResponseCode(403)
- .setBody(TestADLResponseData.getAccessControlException()));
-
- getMockAdlFileSystem()
- .removeAclEntries(new Path("/test1/test2"), entries);
- }
-
- @Test(expected=AccessControlException.class)
- public void testRemoveDefaultAclEntries()
- throws URISyntaxException, IOException {
- getMockServer().enqueue(new MockResponse().setResponseCode(200));
- getMockAdlFileSystem().removeDefaultAcl(new Path("/test1/test2"));
-
- getMockServer().enqueue(new MockResponse().setResponseCode(403)
- .setBody(TestADLResponseData.getAccessControlException()));
-
- getMockAdlFileSystem().removeDefaultAcl(new Path("/test1/test2"));
- }
-
- @Test(expected=AccessControlException.class)
- public void testRemoveAcl() throws URISyntaxException, IOException {
- getMockServer().enqueue(new MockResponse().setResponseCode(200));
- getMockAdlFileSystem().removeAcl(new Path("/test1/test2"));
-
- getMockServer().enqueue(new MockResponse().setResponseCode(403)
- .setBody(TestADLResponseData.getAccessControlException()));
-
- getMockAdlFileSystem().removeAcl(new Path("/test1/test2"));
- }
-
- @Test(expected=AccessControlException.class)
- public void testSetAcl() throws URISyntaxException, IOException {
- getMockServer().enqueue(new MockResponse().setResponseCode(200));
- List<AclEntry> entries = new ArrayList<AclEntry>();
- AclEntry.Builder aclEntryBuilder = new AclEntry.Builder();
- aclEntryBuilder.setName("hadoop");
- aclEntryBuilder.setType(AclEntryType.USER);
- aclEntryBuilder.setPermission(FsAction.ALL);
- aclEntryBuilder.setScope(AclEntryScope.ACCESS);
- entries.add(aclEntryBuilder.build());
-
- aclEntryBuilder.setName("hdfs");
- aclEntryBuilder.setType(AclEntryType.GROUP);
- aclEntryBuilder.setPermission(FsAction.READ_WRITE);
- aclEntryBuilder.setScope(AclEntryScope.DEFAULT);
- entries.add(aclEntryBuilder.build());
-
- getMockAdlFileSystem().setAcl(new Path("/test1/test2"), entries);
-
- getMockServer().enqueue(new MockResponse().setResponseCode(403)
- .setBody(TestADLResponseData.getAccessControlException()));
-
- getMockAdlFileSystem().setAcl(new Path("/test1/test2"), entries);
- }
-
- @Test(expected=AccessControlException.class)
- public void testCheckAccess() throws URISyntaxException, IOException {
- getMockServer().enqueue(new MockResponse().setResponseCode(200));
- getMockAdlFileSystem().access(new Path("/test1/test2"), FsAction.ALL);
-
- getMockServer().enqueue(new MockResponse().setResponseCode(200));
- getMockAdlFileSystem().access(new Path("/test1/test2"), FsAction.EXECUTE);
-
- getMockServer().enqueue(new MockResponse().setResponseCode(200));
- getMockAdlFileSystem().access(new Path("/test1/test2"), FsAction.READ);
-
- getMockServer().enqueue(new MockResponse().setResponseCode(200));
- getMockAdlFileSystem()
- .access(new Path("/test1/test2"), FsAction.READ_EXECUTE);
-
- getMockServer().enqueue(new MockResponse().setResponseCode(200));
- getMockAdlFileSystem()
- .access(new Path("/test1/test2"), FsAction.READ_WRITE);
-
- getMockServer().enqueue(new MockResponse().setResponseCode(200));
- getMockAdlFileSystem().access(new Path("/test1/test2"), FsAction.NONE);
-
- getMockServer().enqueue(new MockResponse().setResponseCode(200));
- getMockAdlFileSystem().access(new Path("/test1/test2"), FsAction.WRITE);
-
- getMockServer().enqueue(new MockResponse().setResponseCode(200));
- getMockAdlFileSystem()
- .access(new Path("/test1/test2"), FsAction.WRITE_EXECUTE);
-
- getMockServer().enqueue(new MockResponse().setResponseCode(403)
- .setBody(TestADLResponseData.getAccessControlException()));
-
- getMockAdlFileSystem()
- .access(new Path("/test1/test2"), FsAction.WRITE_EXECUTE);
- }
-
- @Test(expected=AccessControlException.class)
- public void testSetPermission() throws URISyntaxException, IOException {
- getMockServer().enqueue(new MockResponse().setResponseCode(200));
- getMockAdlFileSystem()
- .setPermission(new Path("/test1/test2"), FsPermission.getDefault());
-
- getMockServer().enqueue(new MockResponse().setResponseCode(403)
- .setBody(TestADLResponseData.getAccessControlException()));
-
- getMockAdlFileSystem()
- .setPermission(new Path("/test1/test2"), FsPermission.getDefault());
- }
-
- @Test(expected=AccessControlException.class)
- public void testSetOwner() throws URISyntaxException, IOException {
- getMockServer().enqueue(new MockResponse().setResponseCode(200));
- getMockAdlFileSystem().setOwner(new Path("/test1/test2"), "hadoop", "hdfs");
-
- getMockServer().enqueue(new MockResponse().setResponseCode(403)
- .setBody(TestADLResponseData.getAccessControlException()));
-
- getMockAdlFileSystem()
- .setOwner(new Path("/test1/test2"), "hadoop", "hdfs");
- }
-
- @Test
- public void getAclStatusAsExpected() throws URISyntaxException, IOException {
- getMockServer().enqueue(new MockResponse().setResponseCode(200)
- .setBody(TestADLResponseData.getGetAclStatusJSONResponse()));
- AclStatus aclStatus = getMockAdlFileSystem()
- .getAclStatus(new Path("/test1/test2"));
- Assert.assertEquals(aclStatus.getGroup(), "supergroup");
- Assert.assertEquals(aclStatus.getOwner(), "hadoop");
- Assert.assertEquals((Short) aclStatus.getPermission().toShort(),
- Short.valueOf("775", 8));
-
- for (AclEntry entry : aclStatus.getEntries()) {
- if (!(entry.toString().equalsIgnoreCase("user:carla:rw-") || entry
- .toString().equalsIgnoreCase("group::r-x"))) {
- Assert.fail("Unexpected entry : " + entry.toString());
- }
- }
- }
-
- @Test(expected=FileNotFoundException.class)
- public void getAclStatusNotExists() throws URISyntaxException, IOException {
- getMockServer().enqueue(new MockResponse().setResponseCode(404)
- .setBody(TestADLResponseData.getFileNotFoundException()));
-
- getMockAdlFileSystem().getAclStatus(new Path("/test1/test2"));
- }
-
- @Test(expected=AccessControlException.class)
- public void testAclStatusDenied() throws URISyntaxException, IOException {
- getMockServer().enqueue(new MockResponse().setResponseCode(403)
- .setBody(TestADLResponseData.getAccessControlException()));
-
- getMockAdlFileSystem().getAclStatus(new Path("/test1/test2"));
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cf88fcd/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAdlRead.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAdlRead.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAdlRead.java
deleted file mode 100644
index 172663c..0000000
--- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAdlRead.java
+++ /dev/null
@@ -1,196 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.adl;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.adl.common.Parallelized;
-import org.apache.hadoop.fs.adl.common.TestDataForRead;
-import org.junit.Assert;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-import java.io.ByteArrayInputStream;
-import java.io.EOFException;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Random;
-
-import static org.apache.hadoop.fs.adl.AdlConfKeys.READ_AHEAD_BUFFER_SIZE_KEY;
-
-/**
- * This class is responsible for stress positional reads vs number of network
- * calls required by to fetch the amount of data. Test does ensure the data
- * integrity and order of the data is maintained.
- */
-@RunWith(Parallelized.class)
-public class TestAdlRead extends AdlMockWebServer {
-
- private TestDataForRead testData;
-
- public TestAdlRead(TestDataForRead testData) {
- Configuration configuration = new Configuration();
- configuration.setInt(READ_AHEAD_BUFFER_SIZE_KEY, 4 * 1024);
- setConf(configuration);
- this.testData = testData;
- }
-
- @Parameterized.Parameters(name = "{index}")
- public static Collection testDataForReadOperation() {
- return Arrays.asList(new Object[][] {
-
- //--------------------------
- // Test Data
- //--------------------------
- {new TestDataForRead("Hello World".getBytes(), 2, 1000, true)},
- {new TestDataForRead(
- ("the problem you appear to be wrestling with is that this doesn't "
- + "display very well. ").getBytes(), 2, 1000, true)},
- {new TestDataForRead(("您的數據是寶貴的資產,以您的組織,並有當前和未來價值。由於這個原因,"
- + "所有的數據應存儲以供將來分析。今天,這往往是不這樣做," + "因為傳統的分析基礎架構的限制,"
- + "像模式的預定義,存儲大數據集和不同的數據筒倉的傳播的成本。"
- + "為了應對這一挑戰,數據湖面概念被引入作為一個企業級存儲庫來存儲所有"
- + "類型的在一個地方收集到的數據。對於運作和探索性分析的目的,所有類型的" + "數據可以定義需求或模式之前被存儲在數據湖。")
- .getBytes(), 2, 1000, true)}, {new TestDataForRead(
- TestADLResponseData.getRandomByteArrayData(4 * 1024), 2, 10, true)},
- {new TestDataForRead(TestADLResponseData.getRandomByteArrayData(100), 2,
- 1000, true)}, {new TestDataForRead(
- TestADLResponseData.getRandomByteArrayData(1 * 1024), 2, 50, true)},
- {new TestDataForRead(
- TestADLResponseData.getRandomByteArrayData(8 * 1024), 3, 10,
- false)}, {new TestDataForRead(
- TestADLResponseData.getRandomByteArrayData(16 * 1024), 5, 10, false)},
- {new TestDataForRead(
- TestADLResponseData.getRandomByteArrayData(32 * 1024), 9, 10,
- false)}, {new TestDataForRead(
- TestADLResponseData.getRandomByteArrayData(64 * 1024), 17, 10,
- false)}});
- }
-
- @Test
- public void testEntireBytes() throws IOException, InterruptedException {
- getMockServer().setDispatcher(testData.getDispatcher());
- FSDataInputStream in = getMockAdlFileSystem().open(new Path("/test"));
- byte[] expectedData = new byte[testData.getActualData().length];
- int n = 0;
- int len = expectedData.length;
- int off = 0;
- while (n < len) {
- int count = in.read(expectedData, off + n, len - n);
- if (count < 0) {
- throw new EOFException();
- }
- n += count;
- }
-
- Assert.assertEquals(expectedData.length, testData.getActualData().length);
- Assert.assertArrayEquals(expectedData, testData.getActualData());
- in.close();
- if (testData.isCheckOfNoOfCalls()) {
- Assert.assertEquals(testData.getExpectedNoNetworkCall(),
- getMockServer().getRequestCount());
- }
- }
-
- @Test
- public void testSeekOperation() throws IOException, InterruptedException {
- getMockServer().setDispatcher(testData.getDispatcher());
- FSDataInputStream in = getMockAdlFileSystem().open(new Path("/test"));
- Random random = new Random();
- for (int i = 0; i < 1000; ++i) {
- int position = random.nextInt(testData.getActualData().length);
- in.seek(position);
- Assert.assertEquals(position, in.getPos());
- Assert.assertEquals(testData.getActualData()[position] & 0xFF, in.read());
- }
- in.close();
- if (testData.isCheckOfNoOfCalls()) {
- Assert.assertEquals(testData.getExpectedNoNetworkCall(),
- getMockServer().getRequestCount());
- }
- }
-
- @Test
- public void testReadServerCalls() throws IOException, InterruptedException {
- getMockServer().setDispatcher(testData.getDispatcher());
- FSDataInputStream in = getMockAdlFileSystem().open(new Path("/test"));
- byte[] expectedData = new byte[testData.getActualData().length];
- in.readFully(expectedData);
- Assert.assertArrayEquals(expectedData, testData.getActualData());
- Assert.assertEquals(testData.getExpectedNoNetworkCall(),
- getMockServer().getRequestCount());
- in.close();
- }
-
- @Test
- public void testReadFully() throws IOException, InterruptedException {
- getMockServer().setDispatcher(testData.getDispatcher());
- FSDataInputStream in = getMockAdlFileSystem().open(new Path("/test"));
- byte[] expectedData = new byte[testData.getActualData().length];
- in.readFully(expectedData);
- Assert.assertArrayEquals(expectedData, testData.getActualData());
-
- in.readFully(0, expectedData);
- Assert.assertArrayEquals(expectedData, testData.getActualData());
-
- in.readFully(0, expectedData, 0, expectedData.length);
- Assert.assertArrayEquals(expectedData, testData.getActualData());
- in.close();
- }
-
- @Test
- public void testRandomPositionalReadUsingReadFully()
- throws IOException, InterruptedException {
- getMockServer().setDispatcher(testData.getDispatcher());
- FSDataInputStream in = getMockAdlFileSystem().open(new Path("/test"));
- ByteArrayInputStream actualData = new ByteArrayInputStream(
- testData.getActualData());
- Random random = new Random();
- for (int i = 0; i < testData.getIntensityOfTest(); ++i) {
- int offset = random.nextInt(testData.getActualData().length);
- int length = testData.getActualData().length - offset;
- byte[] expectedData = new byte[length];
- byte[] actualDataSubset = new byte[length];
- actualData.reset();
- actualData.skip(offset);
- actualData.read(actualDataSubset, 0, length);
-
- in.readFully(offset, expectedData, 0, length);
- Assert.assertArrayEquals(expectedData, actualDataSubset);
- }
-
- for (int i = 0; i < testData.getIntensityOfTest(); ++i) {
- int offset = random.nextInt(testData.getActualData().length);
- int length = random.nextInt(testData.getActualData().length - offset);
- byte[] expectedData = new byte[length];
- byte[] actualDataSubset = new byte[length];
- actualData.reset();
- actualData.skip(offset);
- actualData.read(actualDataSubset, 0, length);
-
- in.readFully(offset, expectedData, 0, length);
- Assert.assertArrayEquals(expectedData, actualDataSubset);
- }
-
- in.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cf88fcd/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestConcurrentDataReadOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestConcurrentDataReadOperations.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestConcurrentDataReadOperations.java
deleted file mode 100644
index b790562..0000000
--- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestConcurrentDataReadOperations.java
+++ /dev/null
@@ -1,299 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.adl;
-
-import com.squareup.okhttp.mockwebserver.Dispatcher;
-import com.squareup.okhttp.mockwebserver.MockResponse;
-import com.squareup.okhttp.mockwebserver.RecordedRequest;
-import okio.Buffer;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.Path;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Random;
-import java.util.UUID;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-/**
- * This class is responsible for testing multiple threads trying to access same
- * or multiple files from the offset.
- */
-@RunWith(Parameterized.class)
-public class TestConcurrentDataReadOperations extends AdlMockWebServer {
- private static final Logger LOG = LoggerFactory
- .getLogger(TestConcurrentDataReadOperations.class);
- private static final Object LOCK = new Object();
- private static FSDataInputStream commonHandle = null;
- private int concurrencyLevel;
-
- public TestConcurrentDataReadOperations(int concurrencyLevel) {
- this.concurrencyLevel = concurrencyLevel;
- }
-
- @Parameterized.Parameters(name = "{index}")
- public static Collection<?> testDataNumberOfConcurrentRun() {
- return Arrays.asList(new Object[][] {{1}, {2}, {3}, {4}, {5}});
- }
-
- public static byte[] getRandomByteArrayData(int size) {
- byte[] b = new byte[size];
- Random rand = new Random();
- rand.nextBytes(b);
- return b;
- }
-
- private void setDispatcher(final ArrayList<CreateTestData> testData) {
- getMockServer().setDispatcher(new Dispatcher() {
- @Override
- public MockResponse dispatch(RecordedRequest recordedRequest)
- throws InterruptedException {
- CreateTestData currentRequest = null;
- for (CreateTestData local : testData) {
- if (recordedRequest.getPath().contains(local.path.toString())) {
- currentRequest = local;
- break;
- }
- }
-
- if (currentRequest == null) {
- new MockResponse().setBody("Request data not found")
- .setResponseCode(501);
- }
-
- if (recordedRequest.getRequestLine().contains("op=GETFILESTATUS")) {
- return new MockResponse().setResponseCode(200).setBody(
- TestADLResponseData
- .getGetFileStatusJSONResponse(currentRequest.data.length));
- }
-
- if (recordedRequest.getRequestLine().contains("op=OPEN")) {
- String request = recordedRequest.getRequestLine();
- int offset = 0;
- int byteCount = 0;
-
- Pattern pattern = Pattern.compile("offset=([0-9]+)");
- Matcher matcher = pattern.matcher(request);
- if (matcher.find()) {
- LOG.debug(matcher.group(1));
- offset = Integer.parseInt(matcher.group(1));
- }
-
- pattern = Pattern.compile("length=([0-9]+)");
- matcher = pattern.matcher(request);
- if (matcher.find()) {
- LOG.debug(matcher.group(1));
- byteCount = Integer.parseInt(matcher.group(1));
- }
-
- Buffer buf = new Buffer();
- buf.write(currentRequest.data, offset,
- Math.min(currentRequest.data.length - offset, byteCount));
- return new MockResponse().setResponseCode(200)
- .setChunkedBody(buf, 4 * 1024 * 1024);
- }
-
- return new MockResponse().setBody("NOT SUPPORTED").setResponseCode(501);
- }
- });
- }
-
- @Before
- public void resetHandle() {
- commonHandle = null;
- }
-
- @Test
- public void testParallelReadOnDifferentStreams()
- throws IOException, InterruptedException, ExecutionException {
-
- ArrayList<CreateTestData> createTestData = new ArrayList<CreateTestData>();
-
- Random random = new Random();
-
- for (int i = 0; i < concurrencyLevel; i++) {
- CreateTestData testData = new CreateTestData();
- testData
- .set(new Path("/test/concurrentRead/" + UUID.randomUUID().toString()),
- getRandomByteArrayData(random.nextInt(1 * 1024 * 1024)));
- createTestData.add(testData);
- }
-
- setDispatcher(createTestData);
-
- ArrayList<ReadTestData> readTestData = new ArrayList<ReadTestData>();
- for (CreateTestData local : createTestData) {
- ReadTestData localReadData = new ReadTestData();
- localReadData.set(local.path, local.data, 0);
- readTestData.add(localReadData);
- }
-
- runReadTest(readTestData, false);
- }
-
- @Test
- public void testParallelReadOnSameStreams()
- throws IOException, InterruptedException, ExecutionException {
- ArrayList<CreateTestData> createTestData = new ArrayList<CreateTestData>();
-
- Random random = new Random();
-
- for (int i = 0; i < 1; i++) {
- CreateTestData testData = new CreateTestData();
- testData
- .set(new Path("/test/concurrentRead/" + UUID.randomUUID().toString()),
- getRandomByteArrayData(1024 * 1024));
- createTestData.add(testData);
- }
-
- setDispatcher(createTestData);
-
- ArrayList<ReadTestData> readTestData = new ArrayList<ReadTestData>();
- ByteArrayInputStream buffered = new ByteArrayInputStream(
- createTestData.get(0).data);
-
- ReadTestData readInitially = new ReadTestData();
- byte[] initialData = new byte[1024 * 1024];
- buffered.read(initialData);
-
- readInitially.set(createTestData.get(0).path, initialData, 0);
- readTestData.add(readInitially);
- runReadTest(readTestData, false);
-
- readTestData.clear();
-
- for (int i = 0; i < concurrencyLevel * 5; i++) {
- ReadTestData localReadData = new ReadTestData();
- int offset = random.nextInt((1024 * 1024) - 1);
- int length = 1024 * 1024 - offset;
- byte[] expectedData = new byte[length];
- buffered.reset();
- buffered.skip(offset);
- buffered.read(expectedData);
- localReadData.set(createTestData.get(0).path, expectedData, offset);
- readTestData.add(localReadData);
- }
-
- runReadTest(readTestData, true);
- }
-
- void runReadTest(ArrayList<ReadTestData> testData, boolean useSameStream)
- throws InterruptedException, ExecutionException {
-
- ExecutorService executor = Executors.newFixedThreadPool(testData.size());
- Future[] subtasks = new Future[testData.size()];
-
- for (int i = 0; i < testData.size(); i++) {
- subtasks[i] = executor.submit(
- new ReadConcurrentRunnable(testData.get(i).data, testData.get(i).path,
- testData.get(i).offset, useSameStream));
- }
-
- executor.shutdown();
-
- // wait until all tasks are finished
- executor.awaitTermination(120, TimeUnit.SECONDS);
-
- for (int i = 0; i < testData.size(); ++i) {
- Assert.assertTrue((Boolean) subtasks[i].get());
- }
- }
-
- class ReadTestData {
- private Path path;
- private byte[] data;
- private int offset;
-
- public void set(Path filePath, byte[] dataToBeRead, int fromOffset) {
- this.path = filePath;
- this.data = dataToBeRead;
- this.offset = fromOffset;
- }
- }
-
- class CreateTestData {
- private Path path;
- private byte[] data;
-
- public void set(Path filePath, byte[] dataToBeWritten) {
- this.path = filePath;
- this.data = dataToBeWritten;
- }
- }
-
- class ReadConcurrentRunnable implements Callable<Boolean> {
- private Path path;
- private int offset;
- private byte[] expectedData;
- private boolean useSameStream;
-
- public ReadConcurrentRunnable(byte[] expectedData, Path path, int offset,
- boolean useSameStream) {
- this.path = path;
- this.offset = offset;
- this.expectedData = expectedData;
- this.useSameStream = useSameStream;
- }
-
- public Boolean call() throws IOException {
- try {
- FSDataInputStream in;
- if (useSameStream) {
- synchronized (LOCK) {
- if (commonHandle == null) {
- commonHandle = getMockAdlFileSystem().open(path);
- }
- in = commonHandle;
- }
- } else {
- in = getMockAdlFileSystem().open(path);
- }
-
- byte[] actualData = new byte[expectedData.length];
- in.readFully(offset, actualData);
- Assert.assertArrayEquals("Path :" + path.toString() + " did not match.",
- expectedData, actualData);
- if (!useSameStream) {
- in.close();
- }
- } catch (IOException e) {
- e.printStackTrace();
- return false;
- }
- return true;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cf88fcd/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestCustomTokenProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestCustomTokenProvider.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestCustomTokenProvider.java
deleted file mode 100644
index 737534c..0000000
--- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestCustomTokenProvider.java
+++ /dev/null
@@ -1,140 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.adl;
-
-import com.squareup.okhttp.mockwebserver.MockResponse;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.adl.common.CustomMockTokenProvider;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.junit.Assert;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.Arrays;
-import java.util.Collection;
-
-import static org.apache.hadoop.fs.adl.AdlConfKeys.ADL_BLOCK_SIZE;
-import static org.apache.hadoop.fs.adl.AdlConfKeys
- .AZURE_AD_TOKEN_PROVIDER_CLASS_KEY;
-import static org.apache.hadoop.fs.adl.AdlConfKeys
- .AZURE_AD_TOKEN_PROVIDER_TYPE_KEY;
-
-/**
- * Test access token provider behaviour with custom token provider and for token
- * provider cache is enabled.
- */
-@RunWith(Parameterized.class)
-public class TestCustomTokenProvider extends AdlMockWebServer {
- private static final long TEN_MINUTES_IN_MILIS = 600000;
- private int backendCallCount;
- private int expectedCallbackToAccessToken;
- private TestableAdlFileSystem[] fileSystems;
- private Class typeOfTokenProviderClass;
- private long expiryFromNow;
- private int fsObjectCount;
-
- public TestCustomTokenProvider(Class typeOfTokenProviderClass,
- long expiryFromNow, int fsObjectCount, int backendCallCount,
- int expectedCallbackToAccessToken)
- throws IllegalAccessException, InstantiationException, URISyntaxException,
- IOException {
- this.typeOfTokenProviderClass = typeOfTokenProviderClass;
- this.expiryFromNow = expiryFromNow;
- this.fsObjectCount = fsObjectCount;
- this.backendCallCount = backendCallCount;
- this.expectedCallbackToAccessToken = expectedCallbackToAccessToken;
- }
-
- @Parameterized.Parameters(name = "{index}")
- public static Collection testDataForTokenProvider() {
- return Arrays.asList(new Object[][] {
- // Data set in order
- // INPUT - CustomTokenProvider class to load
- // INPUT - expiry time in milis. Subtract from current time
- // INPUT - No. of FileSystem object
- // INPUT - No. of backend calls per FileSystem object
- // EXPECTED - Number of callbacks to get token after test finished.
- {CustomMockTokenProvider.class, 0, 1, 1, 1},
- {CustomMockTokenProvider.class, TEN_MINUTES_IN_MILIS, 1, 1, 1},
- {CustomMockTokenProvider.class, TEN_MINUTES_IN_MILIS, 2, 1, 2},
- {CustomMockTokenProvider.class, TEN_MINUTES_IN_MILIS, 10, 10, 10}});
- }
-
- /**
- * Explicitly invoked init so that base class mock server is setup before
- * test data initialization is done.
- *
- * @throws IOException
- * @throws URISyntaxException
- */
- public void init() throws IOException, URISyntaxException {
- Configuration configuration = new Configuration();
- configuration.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY,
- TokenProviderType.Custom);
- configuration.set(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY,
- typeOfTokenProviderClass.getName());
- fileSystems = new TestableAdlFileSystem[fsObjectCount];
- URI uri = new URI("adl://localhost:" + getPort());
-
- for (int i = 0; i < fsObjectCount; ++i) {
- fileSystems[i] = new TestableAdlFileSystem();
- fileSystems[i].initialize(uri, configuration);
-
- ((CustomMockTokenProvider) fileSystems[i].getAzureTokenProvider())
- .setExpiryTimeInMillisAfter(expiryFromNow);
- }
- }
-
- @Test
- public void testCustomTokenManagement()
- throws IOException, URISyntaxException {
- int accessTokenCallbackDuringExec = 0;
- init();
- for (TestableAdlFileSystem tfs : fileSystems) {
- for (int i = 0; i < backendCallCount; ++i) {
- getMockServer().enqueue(new MockResponse().setResponseCode(200)
- .setBody(TestADLResponseData.getGetFileStatusJSONResponse()));
- FileStatus fileStatus = tfs.getFileStatus(new Path("/test1/test2"));
- Assert.assertTrue(fileStatus.isFile());
- Assert.assertEquals("adl://" + getMockServer().getHostName() + ":" +
- getMockServer().getPort() + "/test1/test2",
- fileStatus.getPath().toString());
- Assert.assertEquals(4194304, fileStatus.getLen());
- Assert.assertEquals(ADL_BLOCK_SIZE, fileStatus.getBlockSize());
- Assert.assertEquals(1, fileStatus.getReplication());
- Assert
- .assertEquals(new FsPermission("777"), fileStatus.getPermission());
- Assert.assertEquals("NotSupportYet", fileStatus.getOwner());
- Assert.assertEquals("NotSupportYet", fileStatus.getGroup());
- }
-
- accessTokenCallbackDuringExec += ((CustomMockTokenProvider) tfs
- .getAzureTokenProvider()).getAccessTokenRequestCount();
- }
-
- Assert.assertEquals(expectedCallbackToAccessToken,
- accessTokenCallbackDuringExec);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cf88fcd/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestGetFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestGetFileStatus.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestGetFileStatus.java
deleted file mode 100644
index 95c2363..0000000
--- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestGetFileStatus.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.hadoop.fs.adl;
-
-import com.squareup.okhttp.mockwebserver.MockResponse;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.util.Time;
-import org.junit.Assert;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.URISyntaxException;
-
-import static org.apache.hadoop.fs.adl.AdlConfKeys.ADL_BLOCK_SIZE;
-
-/**
- * This class is responsible for testing local getFileStatus implementation
- * to cover correct parsing of successful and error JSON response
- * from the server.
- * Adls GetFileStatus operation is in detail covered in
- * org.apache.hadoop.fs.adl.live testing package.
- */
-public class TestGetFileStatus extends AdlMockWebServer {
- private static final Logger LOG =
- LoggerFactory.getLogger(TestGetFileStatus.class);
-
- @Test
- public void getFileStatusReturnsAsExpected()
- throws URISyntaxException, IOException {
- getMockServer().enqueue(new MockResponse().setResponseCode(200)
- .setBody(TestADLResponseData.getGetFileStatusJSONResponse()));
- long startTime = Time.monotonicNow();
- Path path = new Path("/test1/test2");
- FileStatus fileStatus = getMockAdlFileSystem().getFileStatus(path);
- long endTime = Time.monotonicNow();
- LOG.debug("Time : " + (endTime - startTime));
- Assert.assertTrue(fileStatus.isFile());
- Assert.assertEquals("adl://" + getMockServer().getHostName() + ":" +
- getMockServer().getPort() + "/test1/test2",
- fileStatus.getPath().toString());
- Assert.assertEquals(4194304, fileStatus.getLen());
- Assert.assertEquals(ADL_BLOCK_SIZE, fileStatus.getBlockSize());
- Assert.assertEquals(1, fileStatus.getReplication());
- Assert.assertEquals(new FsPermission("777"), fileStatus.getPermission());
- Assert.assertEquals("NotSupportYet", fileStatus.getOwner());
- Assert.assertEquals("NotSupportYet", fileStatus.getGroup());
- Assert.assertTrue(path + " should have Acl!", fileStatus.hasAcl());
- Assert.assertFalse(path + " should not be encrypted!",
- fileStatus.isEncrypted());
- Assert.assertFalse(path + " should not be erasure coded!",
- fileStatus.isErasureCoded());
- }
-
- @Test
- public void getFileStatusAclBit() throws URISyntaxException, IOException {
- // With ACLBIT set to true
- getMockServer().enqueue(new MockResponse().setResponseCode(200)
- .setBody(TestADLResponseData.getGetFileStatusJSONResponse(true)));
- long startTime = Time.monotonicNow();
- FileStatus fileStatus = getMockAdlFileSystem()
- .getFileStatus(new Path("/test1/test2"));
- long endTime = Time.monotonicNow();
- LOG.debug("Time : " + (endTime - startTime));
- Assert.assertTrue(fileStatus.isFile());
- Assert.assertTrue(fileStatus.hasAcl());
- Assert.assertTrue(fileStatus.getPermission().getAclBit());
-
- // With ACLBIT set to false
- getMockServer().enqueue(new MockResponse().setResponseCode(200)
- .setBody(TestADLResponseData.getGetFileStatusJSONResponse(false)));
- startTime = Time.monotonicNow();
- fileStatus = getMockAdlFileSystem()
- .getFileStatus(new Path("/test1/test2"));
- endTime = Time.monotonicNow();
- LOG.debug("Time : " + (endTime - startTime));
- Assert.assertTrue(fileStatus.isFile());
- Assert.assertFalse(fileStatus.hasAcl());
- Assert.assertFalse(fileStatus.getPermission().getAclBit());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cf88fcd/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestListStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestListStatus.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestListStatus.java
deleted file mode 100644
index db32476..0000000
--- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestListStatus.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.hadoop.fs.adl;
-
-import com.squareup.okhttp.mockwebserver.MockResponse;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.util.Time;
-import org.junit.Assert;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.URISyntaxException;
-
-/**
- * This class is responsible for testing local listStatus implementation to
- * cover correct parsing of successful and error JSON response from the server.
- * Adls ListStatus functionality is in detail covered in
- * org.apache.hadoop.fs.adl.live testing package.
- */
-public class TestListStatus extends AdlMockWebServer {
-
- private static final Logger LOG = LoggerFactory
- .getLogger(TestListStatus.class);
-
- @Test
- public void listStatusReturnsAsExpected() throws IOException {
- getMockServer().enqueue(new MockResponse().setResponseCode(200)
- .setBody(TestADLResponseData.getListFileStatusJSONResponse(10)));
- long startTime = Time.monotonicNow();
- FileStatus[] ls = getMockAdlFileSystem()
- .listStatus(new Path("/test1/test2"));
- long endTime = Time.monotonicNow();
- LOG.debug("Time : " + (endTime - startTime));
- Assert.assertEquals(10, ls.length);
-
- getMockServer().enqueue(new MockResponse().setResponseCode(200)
- .setBody(TestADLResponseData.getListFileStatusJSONResponse(200)));
- startTime = Time.monotonicNow();
- ls = getMockAdlFileSystem().listStatus(new Path("/test1/test2"));
- endTime = Time.monotonicNow();
- LOG.debug("Time : " + (endTime - startTime));
- Assert.assertEquals(200, ls.length);
-
- getMockServer().enqueue(new MockResponse().setResponseCode(200)
- .setBody(TestADLResponseData.getListFileStatusJSONResponse(2048)));
- startTime = Time.monotonicNow();
- ls = getMockAdlFileSystem().listStatus(new Path("/test1/test2"));
- endTime = Time.monotonicNow();
- LOG.debug("Time : " + (endTime - startTime));
- Assert.assertEquals(2048, ls.length);
- }
-
- @Test
- public void listStatusOnFailure() throws IOException {
- getMockServer().enqueue(new MockResponse().setResponseCode(403).setBody(
- TestADLResponseData.getErrorIllegalArgumentExceptionJSONResponse()));
- FileStatus[] ls = null;
- long startTime = Time.monotonicNow();
- try {
- ls = getMockAdlFileSystem().listStatus(new Path("/test1/test2"));
- } catch (IOException e) {
- Assert.assertTrue(e.getMessage().contains("Invalid"));
- }
- long endTime = Time.monotonicNow();
- LOG.debug("Time : " + (endTime - startTime));
-
- // SDK may increase number of retry attempts before error is propagated
- // to caller. Adding max 10 error responses in the queue to align with SDK.
- for (int i = 0; i < 10; ++i) {
- getMockServer().enqueue(new MockResponse().setResponseCode(500).setBody(
- TestADLResponseData.getErrorInternalServerExceptionJSONResponse()));
- }
-
- startTime = Time.monotonicNow();
- try {
- ls = getMockAdlFileSystem().listStatus(new Path("/test1/test2"));
- } catch (IOException e) {
- Assert.assertTrue(e.getMessage().contains("Internal Server Error"));
- }
- endTime = Time.monotonicNow();
- LOG.debug("Time : " + (endTime - startTime));
- }
-
- @Test
- public void listStatusAcl()
- throws URISyntaxException, IOException {
- // With ACLBIT set to true
- getMockServer().enqueue(new MockResponse().setResponseCode(200)
- .setBody(TestADLResponseData.getListFileStatusJSONResponse(true)));
- FileStatus[] ls = null;
- long startTime = Time.monotonicNow();
- ls = getMockAdlFileSystem()
- .listStatus(new Path("/test1/test2"));
- long endTime = Time.monotonicNow();
- LOG.debug("Time : " + (endTime - startTime));
- for (int i = 0; i < ls.length; i++) {
- Assert.assertTrue(ls[i].isDirectory());
- Assert.assertTrue(ls[i].hasAcl());
- Assert.assertTrue(ls[i].getPermission().getAclBit());
- }
-
- // With ACLBIT set to false
- ls = null;
- getMockServer().enqueue(new MockResponse().setResponseCode(200)
- .setBody(TestADLResponseData.getListFileStatusJSONResponse(false)));
- startTime = Time.monotonicNow();
- ls = getMockAdlFileSystem()
- .listStatus(new Path("/test1/test2"));
- endTime = Time.monotonicNow();
- LOG.debug("Time : " + (endTime - startTime));
- for (int i = 0; i < ls.length; i++) {
- Assert.assertTrue(ls[i].isDirectory());
- Assert.assertFalse(ls[i].hasAcl());
- Assert.assertFalse(ls[i].getPermission().getAclBit());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cf88fcd/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestableAdlFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestableAdlFileSystem.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestableAdlFileSystem.java
deleted file mode 100644
index 4acb39b..0000000
--- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestableAdlFileSystem.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.hadoop.fs.adl;
-
-/**
- * Mock adl file storage subclass to mock adl storage on local http service.
- */
-public class TestableAdlFileSystem extends AdlFileSystem {
- @Override
- protected String getTransportScheme() {
- return "http";
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cf88fcd/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/common/ExpectedResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/common/ExpectedResponse.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/common/ExpectedResponse.java
deleted file mode 100644
index dc8577d..0000000
--- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/common/ExpectedResponse.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.hadoop.fs.adl.common;
-
-import com.squareup.okhttp.mockwebserver.MockResponse;
-
-import java.util.ArrayList;
-
-/**
- * Supporting class to hold expected MockResponse object along with parameters
- * for validation in test methods.
- */
-public class ExpectedResponse {
- private MockResponse response;
- private ArrayList<String> expectedQueryParameters = new ArrayList<String>();
- private int expectedBodySize;
- private String httpRequestType;
-
- public int getExpectedBodySize() {
- return expectedBodySize;
- }
-
- public String getHttpRequestType() {
- return httpRequestType;
- }
-
- public ArrayList<String> getExpectedQueryParameters() {
- return expectedQueryParameters;
- }
-
- public MockResponse getResponse() {
- return response;
- }
-
- ExpectedResponse set(MockResponse mockResponse) {
- this.response = mockResponse;
- return this;
- }
-
- ExpectedResponse addExpectedQueryParam(String param) {
- expectedQueryParameters.add(param);
- return this;
- }
-
- ExpectedResponse addExpectedBodySize(int bodySize) {
- this.expectedBodySize = bodySize;
- return this;
- }
-
- ExpectedResponse addExpectedHttpRequestType(String expectedHttpRequestType) {
- this.httpRequestType = expectedHttpRequestType;
- return this;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cf88fcd/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/common/TestDataForRead.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/common/TestDataForRead.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/common/TestDataForRead.java
deleted file mode 100644
index 509b3f0..0000000
--- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/common/TestDataForRead.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.hadoop.fs.adl.common;
-
-import com.squareup.okhttp.mockwebserver.Dispatcher;
-import com.squareup.okhttp.mockwebserver.MockResponse;
-import com.squareup.okhttp.mockwebserver.RecordedRequest;
-import okio.Buffer;
-import org.apache.hadoop.fs.adl.TestADLResponseData;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-/**
- * Supporting class for mock test to validate Adls read operation.
- */
-public class TestDataForRead {
- private static final Logger LOG = LoggerFactory
- .getLogger(TestDataForRead.class);
-
- private byte[] actualData;
- private ArrayList<ExpectedResponse> responses;
- private Dispatcher dispatcher;
- private int intensityOfTest;
- private boolean checkOfNoOfCalls;
- private int expectedNoNetworkCall;
-
- public TestDataForRead(final byte[] actualData, int expectedNoNetworkCall,
- int intensityOfTest, boolean checkOfNoOfCalls) {
-
- this.checkOfNoOfCalls = checkOfNoOfCalls;
- this.actualData = actualData;
- responses = new ArrayList<ExpectedResponse>();
- this.expectedNoNetworkCall = expectedNoNetworkCall;
- this.intensityOfTest = intensityOfTest;
-
- dispatcher = new Dispatcher() {
- @Override
- public MockResponse dispatch(RecordedRequest recordedRequest)
- throws InterruptedException {
-
- if (recordedRequest.getRequestLine().contains("op=GETFILESTATUS")) {
- return new MockResponse().setResponseCode(200).setBody(
- TestADLResponseData
- .getGetFileStatusJSONResponse(actualData.length));
- }
-
- if (recordedRequest.getRequestLine().contains("op=OPEN")) {
- String request = recordedRequest.getRequestLine();
- int offset = 0;
- int byteCount = 0;
-
- Pattern pattern = Pattern.compile("offset=([0-9]+)");
- Matcher matcher = pattern.matcher(request);
- if (matcher.find()) {
- LOG.debug(matcher.group(1));
- offset = Integer.parseInt(matcher.group(1));
- }
-
- pattern = Pattern.compile("length=([0-9]+)");
- matcher = pattern.matcher(request);
- if (matcher.find()) {
- LOG.debug(matcher.group(1));
- byteCount = Integer.parseInt(matcher.group(1));
- }
-
- Buffer buf = new Buffer();
- buf.write(actualData, offset,
- Math.min(actualData.length - offset, byteCount));
- return new MockResponse().setResponseCode(200)
- .setChunkedBody(buf, 4 * 1024 * 1024);
- }
-
- return new MockResponse().setBody("NOT SUPPORTED").setResponseCode(501);
- }
- };
- }
-
- public boolean isCheckOfNoOfCalls() {
- return checkOfNoOfCalls;
- }
-
- public int getExpectedNoNetworkCall() {
- return expectedNoNetworkCall;
- }
-
- public int getIntensityOfTest() {
- return intensityOfTest;
- }
-
- public byte[] getActualData() {
- return actualData;
- }
-
- public ArrayList<ExpectedResponse> getResponses() {
- return responses;
- }
-
- public Dispatcher getDispatcher() {
- return dispatcher;
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org