You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sz...@apache.org on 2014/09/11 09:24:51 UTC
[11/20] HADOOP-11074. Move s3-related FS connector code to
hadoop-aws. (David S. Wang via Colin Patrick McCabe)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/NativeS3FileSystemContractBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/NativeS3FileSystemContractBaseTest.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/NativeS3FileSystemContractBaseTest.java
new file mode 100644
index 0000000..f6f9ae9
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/NativeS3FileSystemContractBaseTest.java
@@ -0,0 +1,233 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3native;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystemContractBaseTest;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3native.NativeS3FileSystem.NativeS3FsInputStream;
+
+public abstract class NativeS3FileSystemContractBaseTest
+ extends FileSystemContractBaseTest {
+
+ private NativeFileSystemStore store;
+
+ abstract NativeFileSystemStore getNativeFileSystemStore() throws IOException;
+
+ @Override
+ protected void setUp() throws Exception {
+ Configuration conf = new Configuration();
+ store = getNativeFileSystemStore();
+ fs = new NativeS3FileSystem(store);
+ fs.initialize(URI.create(conf.get("test.fs.s3n.name")), conf);
+ }
+
+ @Override
+ protected void tearDown() throws Exception {
+ store.purge("test");
+ super.tearDown();
+ }
+
+ public void testCanonicalName() throws Exception {
+ assertNull("s3n doesn't support security token and shouldn't have canonical name",
+ fs.getCanonicalServiceName());
+ }
+
+ public void testListStatusForRoot() throws Exception {
+ FileStatus[] paths = fs.listStatus(path("/"));
+ assertEquals("Root directory is not empty; ", 0, paths.length);
+
+ Path testDir = path("/test");
+ assertTrue(fs.mkdirs(testDir));
+
+ paths = fs.listStatus(path("/"));
+ assertEquals(1, paths.length);
+ assertEquals(path("/test"), paths[0].getPath());
+ }
+
+ public void testNoTrailingBackslashOnBucket() throws Exception {
+ assertTrue(fs.getFileStatus(new Path(fs.getUri().toString())).isDirectory());
+ }
+
+ private void createTestFiles(String base) throws IOException {
+ store.storeEmptyFile(base + "/file1");
+ store.storeEmptyFile(base + "/dir/file2");
+ store.storeEmptyFile(base + "/dir/file3");
+ }
+
+ public void testDirWithDifferentMarkersWorks() throws Exception {
+
+ for (int i = 0; i < 3; i++) {
+ String base = "test/hadoop" + i;
+ Path path = path("/" + base);
+
+ createTestFiles(base);
+
+ if (i == 0 ) {
+ //do nothing, we are testing correctness with no markers
+ }
+ else if (i == 1) {
+ // test for _$folder$ marker
+ store.storeEmptyFile(base + "_$folder$");
+ store.storeEmptyFile(base + "/dir_$folder$");
+ }
+ else if (i == 2) {
+ // test the end slash file marker
+ store.storeEmptyFile(base + "/");
+ store.storeEmptyFile(base + "/dir/");
+ }
+ else if (i == 3) {
+ // test both markers
+ store.storeEmptyFile(base + "_$folder$");
+ store.storeEmptyFile(base + "/dir_$folder$");
+ store.storeEmptyFile(base + "/");
+ store.storeEmptyFile(base + "/dir/");
+ }
+
+ assertTrue(fs.getFileStatus(path).isDirectory());
+ assertEquals(2, fs.listStatus(path).length);
+ }
+ }
+
+ public void testDeleteWithNoMarker() throws Exception {
+ String base = "test/hadoop";
+ Path path = path("/" + base);
+
+ createTestFiles(base);
+
+ fs.delete(path, true);
+
+ path = path("/test");
+ assertTrue(fs.getFileStatus(path).isDirectory());
+ assertEquals(0, fs.listStatus(path).length);
+ }
+
+ public void testRenameWithNoMarker() throws Exception {
+ String base = "test/hadoop";
+ Path dest = path("/test/hadoop2");
+
+ createTestFiles(base);
+
+ fs.rename(path("/" + base), dest);
+
+ Path path = path("/test");
+ assertTrue(fs.getFileStatus(path).isDirectory());
+ assertEquals(1, fs.listStatus(path).length);
+ assertTrue(fs.getFileStatus(dest).isDirectory());
+ assertEquals(2, fs.listStatus(dest).length);
+ }
+
+ public void testEmptyFile() throws Exception {
+ store.storeEmptyFile("test/hadoop/file1");
+ fs.open(path("/test/hadoop/file1")).close();
+ }
+
+ public void testBlockSize() throws Exception {
+ Path file = path("/test/hadoop/file");
+ createFile(file);
+ assertEquals("Default block size", fs.getDefaultBlockSize(file),
+ fs.getFileStatus(file).getBlockSize());
+
+ // Block size is determined at read time
+ long newBlockSize = fs.getDefaultBlockSize(file) * 2;
+ fs.getConf().setLong("fs.s3n.block.size", newBlockSize);
+ assertEquals("Double default block size", newBlockSize,
+ fs.getFileStatus(file).getBlockSize());
+ }
+
+ public void testRetryOnIoException() throws Exception {
+ class TestInputStream extends InputStream {
+ boolean shouldThrow = false;
+ int throwCount = 0;
+ int pos = 0;
+ byte[] bytes;
+
+ public TestInputStream() {
+ bytes = new byte[256];
+ for (int i = 0; i < 256; i++) {
+ bytes[i] = (byte)i;
+ }
+ }
+
+ @Override
+ public int read() throws IOException {
+ shouldThrow = !shouldThrow;
+ if (shouldThrow) {
+ throwCount++;
+ throw new IOException();
+ }
+ return pos++;
+ }
+
+ @Override
+ public int read(byte[] b, int off, int len) throws IOException {
+ shouldThrow = !shouldThrow;
+ if (shouldThrow) {
+ throwCount++;
+ throw new IOException();
+ }
+
+ int sizeToRead = Math.min(len, 256 - pos);
+ for (int i = 0; i < sizeToRead; i++) {
+ b[i] = bytes[pos + i];
+ }
+ pos += sizeToRead;
+ return sizeToRead;
+ }
+ }
+
+ final InputStream is = new TestInputStream();
+
+ class MockNativeFileSystemStore extends Jets3tNativeFileSystemStore {
+ @Override
+ public InputStream retrieve(String key, long byteRangeStart) throws IOException {
+ return is;
+ }
+ }
+
+ NativeS3FsInputStream stream = new NativeS3FsInputStream(new MockNativeFileSystemStore(), null, is, "");
+
+ // Test reading methods.
+ byte[] result = new byte[256];
+ for (int i = 0; i < 128; i++) {
+ result[i] = (byte)stream.read();
+ }
+ for (int i = 128; i < 256; i += 8) {
+ byte[] temp = new byte[8];
+ int read = stream.read(temp, 0, 8);
+ assertEquals(8, read);
+ System.arraycopy(temp, 0, result, i, 8);
+ }
+
+ // Assert correct
+ for (int i = 0; i < 256; i++) {
+ assertEquals((byte)i, result[i]);
+ }
+
+ // Test to make sure the throw path was exercised.
+ // 144 = 128 + (128 / 8)
+ assertEquals(144, ((TestInputStream)is).throwCount);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/S3NInMemoryFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/S3NInMemoryFileSystem.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/S3NInMemoryFileSystem.java
new file mode 100644
index 0000000..c0ea85b
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/S3NInMemoryFileSystem.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3native;
+
+import org.apache.hadoop.fs.s3native.NativeS3FileSystem;
+import org.apache.hadoop.fs.s3native.InMemoryNativeFileSystemStore;
+
+/**
+ * A helper implementation of {@link NativeS3FileSystem}
+ * without actually connecting to S3 for unit testing.
+ */
+public class S3NInMemoryFileSystem extends NativeS3FileSystem {
+ public S3NInMemoryFileSystem() {
+ super(new InMemoryNativeFileSystemStore());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestInMemoryNativeS3FileSystemContract.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestInMemoryNativeS3FileSystemContract.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestInMemoryNativeS3FileSystemContract.java
new file mode 100644
index 0000000..664d39e
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestInMemoryNativeS3FileSystemContract.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3native;
+
+import java.io.IOException;
+
+public class TestInMemoryNativeS3FileSystemContract
+ extends NativeS3FileSystemContractBaseTest {
+
+ @Override
+ NativeFileSystemStore getNativeFileSystemStore() throws IOException {
+ return new InMemoryNativeFileSystemStore();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestJets3tNativeFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestJets3tNativeFileSystemStore.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestJets3tNativeFileSystemStore.java
new file mode 100644
index 0000000..b1078a4
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestJets3tNativeFileSystemStore.java
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3native;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+
+import static org.junit.Assert.*;
+import static org.junit.Assume.*;
+
+import org.junit.Before;
+import org.junit.After;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.security.DigestInputStream;
+import java.security.DigestOutputStream;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+
+
+public class TestJets3tNativeFileSystemStore {
+ private Configuration conf;
+ private Jets3tNativeFileSystemStore store;
+ private NativeS3FileSystem fs;
+
+ @Before
+ public void setUp() throws Exception {
+ conf = new Configuration();
+ store = new Jets3tNativeFileSystemStore();
+ fs = new NativeS3FileSystem(store);
+ conf.setBoolean("fs.s3n.multipart.uploads.enabled", true);
+ conf.setLong("fs.s3n.multipart.uploads.block.size", 64 * 1024 * 1024);
+ fs.initialize(URI.create(conf.get("test.fs.s3n.name")), conf);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ try {
+ store.purge("test");
+ } catch (Exception e) {}
+ }
+
+ @BeforeClass
+ public static void checkSettings() throws Exception {
+ Configuration conf = new Configuration();
+ assumeNotNull(conf.get("fs.s3n.awsAccessKeyId"));
+ assumeNotNull(conf.get("fs.s3n.awsSecretAccessKey"));
+ assumeNotNull(conf.get("test.fs.s3n.name"));
+ }
+
+ protected void writeRenameReadCompare(Path path, long len)
+ throws IOException, NoSuchAlgorithmException {
+ // If len > fs.s3n.multipart.uploads.block.size,
+ // we'll use a multipart upload copy
+ MessageDigest digest = MessageDigest.getInstance("MD5");
+ OutputStream out = new BufferedOutputStream(
+ new DigestOutputStream(fs.create(path, false), digest));
+ for (long i = 0; i < len; i++) {
+ out.write('Q');
+ }
+ out.flush();
+ out.close();
+
+ assertTrue("Exists", fs.exists(path));
+
+ // Depending on if this file is over 5 GB or not,
+ // rename will cause a multipart upload copy
+ Path copyPath = path.suffix(".copy");
+ fs.rename(path, copyPath);
+
+ assertTrue("Copy exists", fs.exists(copyPath));
+
+ // Download file from S3 and compare the digest against the original
+ MessageDigest digest2 = MessageDigest.getInstance("MD5");
+ InputStream in = new BufferedInputStream(
+ new DigestInputStream(fs.open(copyPath), digest2));
+ long copyLen = 0;
+ while (in.read() != -1) {copyLen++;}
+ in.close();
+
+ assertEquals("Copy length matches original", len, copyLen);
+ assertArrayEquals("Digests match", digest.digest(), digest2.digest());
+ }
+
+ @Test
+ public void testSmallUpload() throws IOException, NoSuchAlgorithmException {
+ // Regular upload, regular copy
+ writeRenameReadCompare(new Path("/test/small"), 16384);
+ }
+
+ @Test
+ public void testMediumUpload() throws IOException, NoSuchAlgorithmException {
+ // Multipart upload, regular copy
+ writeRenameReadCompare(new Path("/test/medium"), 33554432); // 100 MB
+ }
+
+ @Test
+ public void testExtraLargeUpload()
+ throws IOException, NoSuchAlgorithmException {
+ // Multipart upload, multipart copy
+ writeRenameReadCompare(new Path("/test/xlarge"), 5368709121L); // 5GB+1byte
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestS3NInMemoryFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestS3NInMemoryFileSystem.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestS3NInMemoryFileSystem.java
new file mode 100644
index 0000000..b457df2
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestS3NInMemoryFileSystem.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3native;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.URI;
+
+import junit.framework.TestCase;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
+
+public class TestS3NInMemoryFileSystem extends TestCase {
+
+ private static final String TEST_PATH = "s3n://test/data.txt";
+
+ private static final String TEST_DATA = "Sample data for testing.";
+
+ private S3NInMemoryFileSystem fs;
+
+ @Override
+ public void setUp() throws IOException {
+ fs = new S3NInMemoryFileSystem();
+ fs.initialize(URI.create("s3n://test/"), new Configuration());
+ }
+
+ public void testBasicReadWriteIO() throws IOException {
+ FSDataOutputStream writeData = fs.create(new Path(TEST_PATH));
+ writeData.write(TEST_DATA.getBytes());
+ writeData.flush();
+ writeData.close();
+
+ FSDataInputStream readData = fs.open(new Path(TEST_PATH));
+ BufferedReader br = new BufferedReader(new InputStreamReader(readData));
+ String line = "";
+ StringBuffer stringBuffer = new StringBuffer();
+ while ((line = br.readLine()) != null) {
+ stringBuffer.append(line);
+ }
+ br.close();
+
+ assert(TEST_DATA.equals(stringBuffer.toString()));
+ }
+
+ @Override
+ public void tearDown() throws IOException {
+ fs.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-tools/hadoop-aws/src/test/resources/contract/s3n.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/resources/contract/s3n.xml b/hadoop-tools/hadoop-aws/src/test/resources/contract/s3n.xml
new file mode 100644
index 0000000..ab46178
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/resources/contract/s3n.xml
@@ -0,0 +1,95 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<configuration>
+ <!--
+ S3N is a blobstore, with very different behavior than a
+ classic filesystem.
+ -->
+
+ <property>
+ <name>fs.contract.test.root-tests-enabled</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.test.random-seek-count</name>
+ <value>10</value>
+ </property>
+
+ <property>
+ <name>fs.contract.is-blobstore</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.is-case-sensitive</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.rename-returns-false-if-source-missing</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-append</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-atomic-directory-delete</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-atomic-rename</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-block-locality</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-concat</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-seek</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.rejects-seek-past-eof</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-strict-exceptions</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-unix-permissions</name>
+ <value>false</value>
+ </property>
+
+</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec7fcd9/hadoop-tools/hadoop-tools-dist/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-tools-dist/pom.xml b/hadoop-tools/hadoop-tools-dist/pom.xml
index 38043f7..7a01085 100644
--- a/hadoop-tools/hadoop-tools-dist/pom.xml
+++ b/hadoop-tools/hadoop-tools-dist/pom.xml
@@ -85,6 +85,12 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-aws</artifactId>
+ <scope>compile</scope>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-azure</artifactId>
<scope>compile</scope>
<version>${project.version}</version>