You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cn...@apache.org on 2016/10/10 21:04:01 UTC
[3/7] hadoop git commit: HADOOP-13446. Support running isolated unit
tests separate from AWS integration tests. Contributed by Chris Nauroth.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/027d76fa/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AFailureHandling.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AFailureHandling.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AFailureHandling.java
deleted file mode 100644
index 83337f0..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AFailureHandling.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.InvalidRequestException;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-import org.apache.hadoop.fs.contract.AbstractFSContractTestBase;
-import org.apache.hadoop.fs.contract.s3a.S3AContract;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.EOFException;
-import java.io.FileNotFoundException;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
-
-import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.*;
-import static org.apache.hadoop.fs.s3a.S3AUtils.*;
-
-/**
- * Test S3A Failure translation, including a functional test
- * generating errors during stream IO.
- */
-public class TestS3AFailureHandling extends AbstractFSContractTestBase {
- private static final Logger LOG =
- LoggerFactory.getLogger(TestS3AFailureHandling.class);
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new S3AContract(conf);
- }
-
- @Test
- public void testReadFileChanged() throws Throwable {
- describe("overwrite a file with a shorter one during a read, seek");
- final int fullLength = 8192;
- final byte[] fullDataset = dataset(fullLength, 'a', 32);
- final int shortLen = 4096;
- final byte[] shortDataset = dataset(shortLen, 'A', 32);
- final FileSystem fs = getFileSystem();
- final Path testpath = path("readFileToChange.txt");
- // initial write
- writeDataset(fs, testpath, fullDataset, fullDataset.length, 1024, false);
- try(FSDataInputStream instream = fs.open(testpath)) {
- instream.seek(fullLength - 16);
- assertTrue("no data to read", instream.read() >= 0);
- // overwrite
- writeDataset(fs, testpath, shortDataset, shortDataset.length, 1024, true);
- // here the file length is less. Probe the file to see if this is true,
- // with a spin and wait
- eventually(30 *1000, new Callable<Void>() {
- @Override
- public Void call() throws Exception {
- assertEquals(shortLen, fs.getFileStatus(testpath).getLen());
- return null;
- }
- });
- // here length is shorter. Assuming it has propagated to all replicas,
- // the position of the input stream is now beyond the EOF.
- // An attempt to seek backwards to a position greater than the
- // short length will raise an exception from AWS S3, which must be
- // translated into an EOF
-
- instream.seek(shortLen + 1024);
- int c = instream.read();
- assertIsEOF("read()", c);
-
- byte[] buf = new byte[256];
-
- assertIsEOF("read(buffer)", instream.read(buf));
- assertIsEOF("read(offset)",
- instream.read(instream.getPos(), buf, 0, buf.length));
-
- // now do a block read fully, again, backwards from the current pos
- try {
- instream.readFully(shortLen + 512, buf);
- fail("Expected readFully to fail");
- } catch (EOFException expected) {
- LOG.debug("Expected EOF: ", expected);
- }
-
- assertIsEOF("read(offset)",
- instream.read(shortLen + 510, buf, 0, buf.length));
-
- // seek somewhere useful
- instream.seek(shortLen - 256);
-
- // delete the file. Reads must fail
- fs.delete(testpath, false);
-
- try {
- int r = instream.read();
- fail("Expected an exception, got " + r);
- } catch (FileNotFoundException e) {
- // expected
- }
-
- try {
- instream.readFully(2048, buf);
- fail("Expected readFully to fail");
- } catch (FileNotFoundException e) {
- // expected
- }
-
- }
- }
-
- /**
- * Assert that a read operation returned an EOF value.
- * @param operation specific operation
- * @param readResult result
- */
- private void assertIsEOF(String operation, int readResult) {
- assertEquals("Expected EOF from "+ operation
- + "; got char " + (char) readResult, -1, readResult);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/027d76fa/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AFastOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AFastOutputStream.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AFastOutputStream.java
deleted file mode 100644
index e507cf6..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AFastOutputStream.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.contract.ContractTestUtils;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.Timeout;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-
-import java.io.IOException;
-
-/**
- * Tests regular and multi-part upload functionality for S3AFastOutputStream.
- * File sizes are kept small to reduce test duration on slow connections
- */
-public class TestS3AFastOutputStream {
- private FileSystem fs;
-
-
- @Rule
- public Timeout testTimeout = new Timeout(30 * 60 * 1000);
-
- @Before
- public void setUp() throws Exception {
- Configuration conf = new Configuration();
- conf.setLong(Constants.MIN_MULTIPART_THRESHOLD, 5 * 1024 * 1024);
- conf.setInt(Constants.MULTIPART_SIZE, 5 * 1024 * 1024);
- conf.setBoolean(Constants.FAST_UPLOAD, true);
- fs = S3ATestUtils.createTestFileSystem(conf);
- }
-
- @After
- public void tearDown() throws Exception {
- if (fs != null) {
- fs.delete(getTestPath(), true);
- }
- }
-
- protected Path getTestPath() {
- return new Path("/tests3a");
- }
-
- @Test
- public void testRegularUpload() throws IOException {
- ContractTestUtils.createAndVerifyFile(fs, getTestPath(), 1024 * 1024);
- }
-
- @Test
- public void testMultiPartUpload() throws IOException {
- ContractTestUtils.createAndVerifyFile(fs, getTestPath(), 6 * 1024 *
- 1024);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/027d76fa/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AFileOperationCost.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AFileOperationCost.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AFileOperationCost.java
deleted file mode 100644
index 179eb88..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AFileOperationCost.java
+++ /dev/null
@@ -1,277 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-import org.apache.hadoop.fs.contract.AbstractFSContractTestBase;
-import org.apache.hadoop.fs.contract.ContractTestUtils;
-import org.apache.hadoop.fs.contract.s3a.S3AContract;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.net.URI;
-
-import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
-import static org.apache.hadoop.fs.s3a.Statistic.*;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.*;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.MetricDiff;
-
-/**
- * Use metrics to assert about the cost of file status queries.
- * {@link S3AFileSystem#getFileStatus(Path)}.
- */
-public class TestS3AFileOperationCost extends AbstractFSContractTestBase {
-
- private MetricDiff metadataRequests;
- private MetricDiff listRequests;
-
- private static final Logger LOG =
- LoggerFactory.getLogger(TestS3AFileOperationCost.class);
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new S3AContract(conf);
- }
-
- @Override
- public S3AFileSystem getFileSystem() {
- return (S3AFileSystem) super.getFileSystem();
- }
-
- @Override
- public void setup() throws Exception {
- super.setup();
- S3AFileSystem fs = getFileSystem();
- metadataRequests = new MetricDiff(fs, OBJECT_METADATA_REQUESTS);
- listRequests = new MetricDiff(fs, OBJECT_LIST_REQUESTS);
- }
-
- @Test
- public void testCostOfGetFileStatusOnFile() throws Throwable {
- describe("performing getFileStatus on a file");
- Path simpleFile = path("simple.txt");
- S3AFileSystem fs = getFileSystem();
- touch(fs, simpleFile);
- resetMetricDiffs();
- S3AFileStatus status = fs.getFileStatus(simpleFile);
- assertTrue("not a file: " + status, status.isFile());
- metadataRequests.assertDiffEquals(1);
- listRequests.assertDiffEquals(0);
- }
-
- private void resetMetricDiffs() {
- reset(metadataRequests, listRequests);
- }
-
- @Test
- public void testCostOfGetFileStatusOnEmptyDir() throws Throwable {
- describe("performing getFileStatus on an empty directory");
- S3AFileSystem fs = getFileSystem();
- Path dir = path("empty");
- fs.mkdirs(dir);
- resetMetricDiffs();
- S3AFileStatus status = fs.getFileStatus(dir);
- assertTrue("not empty: " + status, status.isEmptyDirectory());
- metadataRequests.assertDiffEquals(2);
- listRequests.assertDiffEquals(0);
- }
-
- @Test
- public void testCostOfGetFileStatusOnMissingFile() throws Throwable {
- describe("performing getFileStatus on a missing file");
- S3AFileSystem fs = getFileSystem();
- Path path = path("missing");
- resetMetricDiffs();
- try {
- S3AFileStatus status = fs.getFileStatus(path);
- fail("Got a status back from a missing file path " + status);
- } catch (FileNotFoundException expected) {
- // expected
- }
- metadataRequests.assertDiffEquals(2);
- listRequests.assertDiffEquals(1);
- }
-
- @Test
- public void testCostOfGetFileStatusOnMissingSubPath() throws Throwable {
- describe("performing getFileStatus on a missing file");
- S3AFileSystem fs = getFileSystem();
- Path path = path("missingdir/missingpath");
- resetMetricDiffs();
- try {
- S3AFileStatus status = fs.getFileStatus(path);
- fail("Got a status back from a missing file path " + status);
- } catch (FileNotFoundException expected) {
- // expected
- }
- metadataRequests.assertDiffEquals(2);
- listRequests.assertDiffEquals(1);
- }
-
- @Test
- public void testCostOfGetFileStatusOnNonEmptyDir() throws Throwable {
- describe("performing getFileStatus on a non-empty directory");
- S3AFileSystem fs = getFileSystem();
- Path dir = path("empty");
- fs.mkdirs(dir);
- Path simpleFile = new Path(dir, "simple.txt");
- touch(fs, simpleFile);
- resetMetricDiffs();
- S3AFileStatus status = fs.getFileStatus(dir);
- if (status.isEmptyDirectory()) {
- // erroneous state
- String fsState = fs.toString();
- fail("FileStatus says directory isempty: " + status
- + "\n" + ContractTestUtils.ls(fs, dir)
- + "\n" + fsState);
- }
- metadataRequests.assertDiffEquals(2);
- listRequests.assertDiffEquals(1);
- }
-
- @Test
- public void testCostOfCopyFromLocalFile() throws Throwable {
- describe("testCostOfCopyFromLocalFile");
- String testDirProp = System.getProperty("test.build.data",
- "target" + File.separator + "test" + File.separator + "data");
- File localTestDir = new File(testDirProp, "tmp").getAbsoluteFile();
- localTestDir.mkdirs();
- File tmpFile = File.createTempFile("tests3acost", ".txt",
- localTestDir);
- tmpFile.delete();
- try {
- URI localFileURI = tmpFile.toURI();
- FileSystem localFS = FileSystem.get(localFileURI,
- getFileSystem().getConf());
- Path localPath = new Path(localFileURI);
- int len = 10 * 1024;
- byte[] data = dataset(len, 'A', 'Z');
- writeDataset(localFS, localPath, data, len, 1024, true);
- S3AFileSystem s3a = getFileSystem();
- MetricDiff copyLocalOps = new MetricDiff(s3a,
- INVOCATION_COPY_FROM_LOCAL_FILE);
- MetricDiff putRequests = new MetricDiff(s3a,
- OBJECT_PUT_REQUESTS);
- MetricDiff putBytes = new MetricDiff(s3a,
- OBJECT_PUT_BYTES);
-
- Path remotePath = path("copied");
- s3a.copyFromLocalFile(false, true, localPath, remotePath);
- verifyFileContents(s3a, remotePath, data);
- copyLocalOps.assertDiffEquals(1);
- putRequests.assertDiffEquals(1);
- putBytes.assertDiffEquals(len);
- // print final stats
- LOG.info("Filesystem {}", s3a);
- } finally {
- tmpFile.delete();
- }
- }
-
- private void reset(MetricDiff... diffs) {
- for (MetricDiff diff : diffs) {
- diff.reset();
- }
- }
-
- @Test
- public void testFakeDirectoryDeletion() throws Throwable {
- describe("Verify whether create file works after renaming a file. "
- + "In S3, rename deletes any fake directories as a part of "
- + "clean up activity");
- S3AFileSystem fs = getFileSystem();
- Path srcBaseDir = path("src");
- mkdirs(srcBaseDir);
- MetricDiff deleteRequests =
- new MetricDiff(fs, Statistic.OBJECT_DELETE_REQUESTS);
- MetricDiff directoriesDeleted =
- new MetricDiff(fs, Statistic.DIRECTORIES_DELETED);
- MetricDiff fakeDirectoriesDeleted =
- new MetricDiff(fs, Statistic.FAKE_DIRECTORIES_DELETED);
- MetricDiff directoriesCreated =
- new MetricDiff(fs, Statistic.DIRECTORIES_CREATED);
-
- Path srcDir = new Path(srcBaseDir, "1/2/3/4/5/6");
- Path srcFilePath = new Path(srcDir, "source.txt");
- int srcDirDepth = directoriesInPath(srcDir);
- // one dir created, one removed
- mkdirs(srcDir);
- String state = "after mkdir(srcDir)";
- directoriesCreated.assertDiffEquals(state, 1);
-/* TODO: uncomment once HADOOP-13222 is in
- deleteRequests.assertDiffEquals(state, 1);
- directoriesDeleted.assertDiffEquals(state, 0);
- fakeDirectoriesDeleted.assertDiffEquals(state, srcDirDepth);
-*/
- reset(deleteRequests, directoriesCreated, directoriesDeleted,
- fakeDirectoriesDeleted);
-
- // creating a file should trigger demise of the src dir
- touch(fs, srcFilePath);
- state = "after touch(fs, srcFilePath)";
- deleteRequests.assertDiffEquals(state, 1);
- directoriesCreated.assertDiffEquals(state, 0);
- directoriesDeleted.assertDiffEquals(state, 0);
- fakeDirectoriesDeleted.assertDiffEquals(state, srcDirDepth);
-
- reset(deleteRequests, directoriesCreated, directoriesDeleted,
- fakeDirectoriesDeleted);
-
- Path destBaseDir = path("dest");
- Path destDir = new Path(destBaseDir, "1/2/3/4/5/6");
- Path destFilePath = new Path(destDir, "dest.txt");
- mkdirs(destDir);
- state = "after mkdir(destDir)";
-
- int destDirDepth = directoriesInPath(destDir);
- directoriesCreated.assertDiffEquals(state, 1);
-/* TODO: uncomment once HADOOP-13222 is in
- deleteRequests.assertDiffEquals(state,1);
- directoriesDeleted.assertDiffEquals(state,0);
- fakeDirectoriesDeleted.assertDiffEquals(state,destDirDepth);
-*/
- reset(deleteRequests, directoriesCreated, directoriesDeleted,
- fakeDirectoriesDeleted);
-
- fs.rename(srcFilePath, destFilePath);
- state = "after rename(srcFilePath, destFilePath)";
- directoriesCreated.assertDiffEquals(state, 1);
- // one for the renamed file, one for the parent
- deleteRequests.assertDiffEquals(state, 2);
- directoriesDeleted.assertDiffEquals(state, 0);
- fakeDirectoriesDeleted.assertDiffEquals(state, destDirDepth);
-
- reset(deleteRequests, directoriesCreated, directoriesDeleted,
- fakeDirectoriesDeleted);
-
- assertIsFile(destFilePath);
- assertIsDirectory(srcDir);
- }
-
- private int directoriesInPath(Path path) {
- return path.isRoot() ? 0 : 1 + directoriesInPath(path.getParent());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/027d76fa/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AFileSystemContract.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AFileSystemContract.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AFileSystemContract.java
deleted file mode 100644
index 6315a80..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AFileSystemContract.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystemContractBaseTest;
-import org.apache.hadoop.fs.Path;
-
-/**
- * Tests a live S3 system. If your keys and bucket aren't specified, all tests
- * are marked as passed.
- *
- * This uses BlockJUnit4ClassRunner because FileSystemContractBaseTest from
- * TestCase which uses the old Junit3 runner that doesn't ignore assumptions
- * properly making it impossible to skip the tests if we don't have a valid
- * bucket.
- **/
-public class TestS3AFileSystemContract extends FileSystemContractBaseTest {
-
- protected static final Logger LOG =
- LoggerFactory.getLogger(TestS3AFileSystemContract.class);
-
- @Override
- public void setUp() throws Exception {
- Configuration conf = new Configuration();
-
- fs = S3ATestUtils.createTestFileSystem(conf);
- super.setUp();
- }
-
- @Override
- protected void tearDown() throws Exception {
- if (fs != null) {
- fs.delete(path("test"), true);
- }
- super.tearDown();
- }
-
- @Override
- public void testMkdirsWithUmask() throws Exception {
- // not supported
- }
-
- @Override
- public void testRenameFileAsExistingFile() throws Exception {
- if (!renameSupported()) return;
-
- Path src = path("/test/hadoop/file");
- createFile(src);
- Path dst = path("/test/new/newfile");
- createFile(dst);
- // s3 doesn't support rename option
- // rename-overwrites-dest is always allowed.
- rename(src, dst, true, false, true);
- }
-
- @Override
- public void testRenameDirectoryAsExistingDirectory() throws Exception {
- if (!renameSupported()) {
- return;
- }
-
- Path src = path("/test/hadoop/dir");
- fs.mkdirs(src);
- createFile(path("/test/hadoop/dir/file1"));
- createFile(path("/test/hadoop/dir/subdir/file2"));
-
- Path dst = path("/test/new/newdir");
- fs.mkdirs(dst);
- rename(src, dst, true, false, true);
- assertFalse("Nested file1 exists",
- fs.exists(path("/test/hadoop/dir/file1")));
- assertFalse("Nested file2 exists",
- fs.exists(path("/test/hadoop/dir/subdir/file2")));
- assertTrue("Renamed nested file1 exists",
- fs.exists(path("/test/new/newdir/file1")));
- assertTrue("Renamed nested exists",
- fs.exists(path("/test/new/newdir/subdir/file2")));
- }
-
-// @Override
- public void testMoveDirUnderParent() throws Throwable {
- // not support because
- // Fails if dst is a directory that is not empty.
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/027d76fa/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ATemporaryCredentials.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ATemporaryCredentials.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ATemporaryCredentials.java
deleted file mode 100644
index 957dfb9..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ATemporaryCredentials.java
+++ /dev/null
@@ -1,148 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import java.io.IOException;
-import java.net.URI;
-
-import com.amazonaws.auth.AWSCredentials;
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClient;
-import com.amazonaws.services.securitytoken.model.GetSessionTokenRequest;
-import com.amazonaws.services.securitytoken.model.GetSessionTokenResult;
-import com.amazonaws.services.securitytoken.model.Credentials;
-
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-import org.apache.hadoop.fs.contract.AbstractFSContractTestBase;
-import org.apache.hadoop.fs.contract.s3a.S3AContract;
-import org.apache.hadoop.fs.s3native.S3xLoginHelper;
-import org.apache.hadoop.conf.Configuration;
-
-import org.junit.Test;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
-import static org.apache.hadoop.fs.s3a.Constants.*;
-
-/**
- * Tests use of temporary credentials (for example, AWS STS & S3).
- * This test extends a class that "does things to the root directory", and
- * should only be used against transient filesystems where you don't care about
- * the data.
- */
-public class TestS3ATemporaryCredentials extends AbstractFSContractTestBase {
- public static final String TEST_STS_ENABLED = "test.fs.s3a.sts.enabled";
- public static final String TEST_STS_ENDPOINT = "test.fs.s3a.sts.endpoint";
-
- private static final Logger LOG =
- LoggerFactory.getLogger(TestS3ATemporaryCredentials.class);
-
- private static final String PROVIDER_CLASS
- = TemporaryAWSCredentialsProvider.NAME;
-
- private static final long TEST_FILE_SIZE = 1024;
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new S3AContract(conf);
- }
-
- /**
- * Test use of STS for requesting temporary credentials.
- *
- * The property test.sts.endpoint can be set to point this at different
- * STS endpoints. This test will use the AWS credentials (if provided) for
- * S3A tests to request temporary credentials, then attempt to use those
- * credentials instead.
- *
- * @throws IOException
- */
- @Test
- public void testSTS() throws IOException {
- Configuration conf = getContract().getConf();
- if (!conf.getBoolean(TEST_STS_ENABLED, true)) {
- skip("STS functional tests disabled");
- }
-
- S3xLoginHelper.Login login = S3AUtils.getAWSAccessKeys(
- URI.create("s3a://foobar"), conf);
- if (!login.hasLogin()) {
- skip("testSTS disabled because AWS credentials not configured");
- }
- AWSCredentialsProvider parentCredentials = new BasicAWSCredentialsProvider(
- login.getUser(), login.getPassword());
-
- String stsEndpoint = conf.getTrimmed(TEST_STS_ENDPOINT, "");
- AWSSecurityTokenServiceClient stsClient;
- stsClient = new AWSSecurityTokenServiceClient(parentCredentials);
- if (!stsEndpoint.isEmpty()) {
- LOG.debug("STS Endpoint ={}", stsEndpoint);
- stsClient.setEndpoint(stsEndpoint);
- }
- GetSessionTokenRequest sessionTokenRequest = new GetSessionTokenRequest();
- sessionTokenRequest.setDurationSeconds(900);
- GetSessionTokenResult sessionTokenResult;
- sessionTokenResult = stsClient.getSessionToken(sessionTokenRequest);
- Credentials sessionCreds = sessionTokenResult.getCredentials();
-
- String childAccessKey = sessionCreds.getAccessKeyId();
- conf.set(ACCESS_KEY, childAccessKey);
- String childSecretKey = sessionCreds.getSecretAccessKey();
- conf.set(SECRET_KEY, childSecretKey);
- String sessionToken = sessionCreds.getSessionToken();
- conf.set(SESSION_TOKEN, sessionToken);
-
- conf.set(AWS_CREDENTIALS_PROVIDER, PROVIDER_CLASS);
-
- try(S3AFileSystem fs = S3ATestUtils.createTestFileSystem(conf)) {
- createAndVerifyFile(fs, path("testSTS"), TEST_FILE_SIZE);
- }
-
- // now create an invalid set of credentials by changing the session
- // token
- conf.set(SESSION_TOKEN, "invalid-" + sessionToken);
- try (S3AFileSystem fs = S3ATestUtils.createTestFileSystem(conf)) {
- createAndVerifyFile(fs, path("testSTSInvalidToken"), TEST_FILE_SIZE);
- fail("Expected an access exception, but file access to "
- + fs.getUri() + " was allowed: " + fs);
- } catch (AWSS3IOException ex) {
- LOG.info("Expected Exception: {}", ex.toString());
- LOG.debug("Expected Exception: {}", ex, ex);
- }
- }
-
- @Test
- public void testTemporaryCredentialValidation() throws Throwable {
- Configuration conf = new Configuration();
- conf.set(ACCESS_KEY, "accesskey");
- conf.set(SECRET_KEY, "secretkey");
- conf.set(SESSION_TOKEN, "");
- TemporaryAWSCredentialsProvider provider
- = new TemporaryAWSCredentialsProvider(getFileSystem().getUri(), conf);
- try {
- AWSCredentials credentials = provider.getCredentials();
- fail("Expected a CredentialInitializationException,"
- + " got " + credentials);
- } catch (CredentialInitializationException expected) {
- // expected
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/027d76fa/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContext.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContext.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContext.java
new file mode 100644
index 0000000..7e4273a
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContext.java
@@ -0,0 +1,23 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.s3a.fileContext;
+
+import org.apache.hadoop.fs.TestFileContext;
+
+/**
+ * Implementation of TestFileContext for S3a.
+ */
+public class ITestS3AFileContext extends TestFileContext{
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/027d76fa/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextCreateMkdir.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextCreateMkdir.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextCreateMkdir.java
new file mode 100644
index 0000000..4b8d4bb
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextCreateMkdir.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.s3a.fileContext;
+
+import java.io.IOException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileContextCreateMkdirBaseTest;
+import org.apache.hadoop.fs.s3a.S3ATestUtils;
+import org.junit.Before;
+
+/**
+ * Extends FileContextCreateMkdirBaseTest for a S3a FileContext.
+ */
+public class ITestS3AFileContextCreateMkdir
+ extends FileContextCreateMkdirBaseTest {
+
+ @Before
+ public void setUp() throws IOException, Exception {
+ Configuration conf = new Configuration();
+ fc = S3ATestUtils.createTestFileContext(conf);
+ super.setUp();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/027d76fa/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextMainOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextMainOperations.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextMainOperations.java
new file mode 100644
index 0000000..7247c16
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextMainOperations.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.s3a.fileContext;
+
+import java.io.IOException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileContextMainOperationsBaseTest;
+import org.apache.hadoop.fs.s3a.S3ATestUtils;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+/**
+ * S3A implementation of FileContextMainOperationsBaseTest.
+ */
+public class ITestS3AFileContextMainOperations
+ extends FileContextMainOperationsBaseTest {
+
+ @Before
+ public void setUp() throws IOException, Exception {
+ Configuration conf = new Configuration();
+ fc = S3ATestUtils.createTestFileContext(conf);
+ super.setUp();
+ }
+
+ @Override
+ protected boolean listCorruptedBlocksSupported() {
+ return false;
+ }
+
+ @Test
+ @Ignore
+ public void testCreateFlagAppendExistingFile() throws IOException {
+ //append not supported, so test removed
+ }
+
+ @Test
+ @Ignore
+ public void testCreateFlagCreateAppendExistingFile() throws IOException {
+ //append not supported, so test removed
+ }
+
+ @Test
+ @Ignore
+ public void testSetVerifyChecksum() throws IOException {
+ //checksums ignored, so test removed
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/027d76fa/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextStatistics.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextStatistics.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextStatistics.java
new file mode 100644
index 0000000..5e88aba
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextStatistics.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.s3a.fileContext;
+
+import java.net.URI;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FCStatisticsBaseTest;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.s3a.S3ATestUtils;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+
+/**
+ * S3a implementation of FCStatisticsBaseTest.
+ */
+public class ITestS3AFileContextStatistics extends FCStatisticsBaseTest {
+
+ @Before
+ public void setUp() throws Exception {
+ Configuration conf = new Configuration();
+ fc = S3ATestUtils.createTestFileContext(conf);
+ fc.mkdir(fileContextTestHelper.getTestRootPath(fc, "test"),
+ FileContext.DEFAULT_PERM, true);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ fc.delete(fileContextTestHelper.getTestRootPath(fc, "test"), true);
+ }
+
+ @Override
+ protected void verifyReadBytes(FileSystem.Statistics stats) {
+ // one blockSize for read, one for pread
+ Assert.assertEquals(2 * blockSize, stats.getBytesRead());
+ }
+
+ @Override
+ protected void verifyWrittenBytes(FileSystem.Statistics stats) {
+ //No extra bytes are written
+ Assert.assertEquals(blockSize, stats.getBytesWritten());
+ }
+
+ @Override
+ protected URI getFsUri() {
+ return fc.getHomeDirectory().toUri();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/027d76fa/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextURI.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextURI.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextURI.java
new file mode 100644
index 0000000..fff1fcb
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextURI.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.s3a.fileContext;
+
+import java.io.IOException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileContextURIBase;
+import org.apache.hadoop.fs.s3a.S3ATestUtils;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+/**
+ * S3a implementation of FileContextURIBase.
+ */
+public class ITestS3AFileContextURI extends FileContextURIBase {
+
+ @Before
+ public void setUp() throws IOException, Exception {
+ Configuration conf = new Configuration();
+ fc1 = S3ATestUtils.createTestFileContext(conf);
+ fc2 = S3ATestUtils.createTestFileContext(conf); //different object, same FS
+ super.setUp();
+ }
+
+ @Test
+ @Ignore
+ public void testFileStatus() throws IOException {
+ // test disabled
+ // (the statistics tested with this method are not relevant for an S3FS)
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/027d76fa/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextUtil.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextUtil.java
new file mode 100644
index 0000000..d0312ba
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextUtil.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.s3a.fileContext;
+
+import java.io.IOException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileContextUtilBase;
+import org.apache.hadoop.fs.s3a.S3ATestUtils;
+import org.junit.Before;
+
+/**
+ * S3A implementation of FileContextUtilBase.
+ */
+public class ITestS3AFileContextUtil extends FileContextUtilBase {
+
+ @Before
+ public void setUp() throws IOException, Exception {
+ Configuration conf = new Configuration();
+ fc = S3ATestUtils.createTestFileContext(conf);
+ super.setUp();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/027d76fa/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContext.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContext.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContext.java
deleted file mode 100644
index 34d78a5..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContext.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.s3a.fileContext;
-
-import org.apache.hadoop.fs.TestFileContext;
-
-/**
- * Implementation of TestFileContext for S3a
- */
-public class TestS3AFileContext extends TestFileContext{
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/027d76fa/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextCreateMkdir.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextCreateMkdir.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextCreateMkdir.java
deleted file mode 100644
index b0c4d84..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextCreateMkdir.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/**
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.s3a.fileContext;
-
-import java.io.IOException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileContextCreateMkdirBaseTest;
-import org.apache.hadoop.fs.s3a.S3ATestUtils;
-import org.junit.Before;
-
-/**
- * Extends FileContextCreateMkdirBaseTest for a S3a FileContext
- */
-public class TestS3AFileContextCreateMkdir
- extends FileContextCreateMkdirBaseTest {
-
- @Before
- public void setUp() throws IOException, Exception {
- Configuration conf = new Configuration();
- fc = S3ATestUtils.createTestFileContext(conf);
- super.setUp();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/027d76fa/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextMainOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextMainOperations.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextMainOperations.java
deleted file mode 100644
index 4d200d1..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextMainOperations.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/**
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.s3a.fileContext;
-
-import java.io.IOException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileContextMainOperationsBaseTest;
-import org.apache.hadoop.fs.s3a.S3ATestUtils;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-/**
- * S3A implementation of FileContextMainOperationsBaseTest
- */
-public class TestS3AFileContextMainOperations
- extends FileContextMainOperationsBaseTest {
-
- @Before
- public void setUp() throws IOException, Exception {
- Configuration conf = new Configuration();
- fc = S3ATestUtils.createTestFileContext(conf);
- super.setUp();
- }
-
- @Override
- protected boolean listCorruptedBlocksSupported() {
- return false;
- }
-
- @Test
- @Ignore
- public void testCreateFlagAppendExistingFile() throws IOException {
- //append not supported, so test removed
- }
-
- @Test
- @Ignore
- public void testCreateFlagCreateAppendExistingFile() throws IOException {
- //append not supported, so test removed
- }
-
- @Test
- @Ignore
- public void testSetVerifyChecksum() throws IOException {
- //checksums ignored, so test removed
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/027d76fa/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextStatistics.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextStatistics.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextStatistics.java
deleted file mode 100644
index a9f4848..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextStatistics.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/**
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.s3a.fileContext;
-
-import java.net.URI;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FCStatisticsBaseTest;
-import org.apache.hadoop.fs.FileContext;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.s3a.S3ATestUtils;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-
-/**
- * S3a implementation of FCStatisticsBaseTest
- */
-public class TestS3AFileContextStatistics extends FCStatisticsBaseTest {
-
- @Before
- public void setUp() throws Exception {
- Configuration conf = new Configuration();
- fc = S3ATestUtils.createTestFileContext(conf);
- fc.mkdir(fileContextTestHelper.getTestRootPath(fc, "test"), FileContext.DEFAULT_PERM, true);
- }
-
- @After
- public void tearDown() throws Exception {
- fc.delete(fileContextTestHelper.getTestRootPath(fc, "test"), true);
- }
-
- @Override
- protected void verifyReadBytes(FileSystem.Statistics stats) {
- // one blockSize for read, one for pread
- Assert.assertEquals(2 * blockSize, stats.getBytesRead());
- }
-
- @Override
- protected void verifyWrittenBytes(FileSystem.Statistics stats) {
- //No extra bytes are written
- Assert.assertEquals(blockSize, stats.getBytesWritten());
- }
-
- @Override
- protected URI getFsUri() {
- return fc.getHomeDirectory().toUri();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/027d76fa/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextURI.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextURI.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextURI.java
deleted file mode 100644
index 3da7b19..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextURI.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/**
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.s3a.fileContext;
-
-import java.io.IOException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileContextURIBase;
-import org.apache.hadoop.fs.s3a.S3ATestUtils;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-/**
- * S3a implementation of FileContextURIBase
- */
-public class TestS3AFileContextURI extends FileContextURIBase {
-
- @Before
- public void setUp() throws IOException, Exception {
- Configuration conf = new Configuration();
- fc1 = S3ATestUtils.createTestFileContext(conf);
- fc2 = S3ATestUtils.createTestFileContext(conf); //different object, same FS
- super.setUp();
- }
-
- @Test
- @Ignore
- public void testFileStatus() throws IOException {
- //test disabled (the statistics tested with this method are not relevant for an S3FS)
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/027d76fa/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextUtil.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextUtil.java
deleted file mode 100644
index 666f4c2..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextUtil.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/**
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.s3a.fileContext;
-
-import java.io.IOException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileContextUtilBase;
-import org.apache.hadoop.fs.s3a.S3ATestUtils;
-import org.junit.Before;
-
-/**
- * S3A implementation of FileContextUtilBase
- */
-public class TestS3AFileContextUtil extends FileContextUtilBase {
-
- @Before
- public void setUp() throws IOException, Exception {
- Configuration conf = new Configuration();
- fc = S3ATestUtils.createTestFileContext(conf);
- super.setUp();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/027d76fa/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADeleteFilesOneByOne.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADeleteFilesOneByOne.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADeleteFilesOneByOne.java
new file mode 100644
index 0000000..a375664
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADeleteFilesOneByOne.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.scale;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.s3a.Constants;
+import org.junit.Test;
+
+import java.io.IOException;
+
+/**
+ * Tests file deletion with multi-delete disabled.
+ */
+public class ITestS3ADeleteFilesOneByOne extends ITestS3ADeleteManyFiles {
+
+ @Override
+ protected Configuration createConfiguration() {
+ Configuration configuration = super.createConfiguration();
+ configuration.setBoolean(Constants.ENABLE_MULTI_DELETE, false);
+ return configuration;
+ }
+
+ @Override
+ @Test
+ public void testOpenCreate() throws IOException {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/027d76fa/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADeleteManyFiles.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADeleteManyFiles.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADeleteManyFiles.java
new file mode 100644
index 0000000..542dc12
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADeleteManyFiles.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.scale;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+/**
+ * Test some scalable operations related to file renaming and deletion.
+ */
+public class ITestS3ADeleteManyFiles extends S3AScaleTestBase {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ITestS3ADeleteManyFiles.class);
+
+ @Test
+ public void testBulkRenameAndDelete() throws Throwable {
+ final Path scaleTestDir = getTestPath();
+ final Path srcDir = new Path(scaleTestDir, "src");
+ final Path finalDir = new Path(scaleTestDir, "final");
+ final long count = getOperationCount();
+ ContractTestUtils.rm(fs, scaleTestDir, true, false);
+
+ fs.mkdirs(srcDir);
+ fs.mkdirs(finalDir);
+
+ int testBufferSize = fs.getConf()
+ .getInt(ContractTestUtils.IO_CHUNK_BUFFER_SIZE,
+ ContractTestUtils.DEFAULT_IO_CHUNK_BUFFER_SIZE);
+ // use Executor to speed up file creation
+ ExecutorService exec = Executors.newFixedThreadPool(16);
+ final ExecutorCompletionService<Boolean> completionService =
+ new ExecutorCompletionService<>(exec);
+ try {
+ final byte[] data = ContractTestUtils.dataset(testBufferSize, 'a', 'z');
+
+ for (int i = 0; i < count; ++i) {
+ final String fileName = "foo-" + i;
+ completionService.submit(new Callable<Boolean>() {
+ @Override
+ public Boolean call() throws IOException {
+ ContractTestUtils.createFile(fs, new Path(srcDir, fileName),
+ false, data);
+ return fs.exists(new Path(srcDir, fileName));
+ }
+ });
+ }
+ for (int i = 0; i < count; ++i) {
+ final Future<Boolean> future = completionService.take();
+ try {
+ if (!future.get()) {
+ LOG.warn("cannot create file");
+ }
+ } catch (ExecutionException e) {
+ LOG.warn("Error while uploading file", e.getCause());
+ throw e;
+ }
+ }
+ } finally {
+ exec.shutdown();
+ }
+
+ int nSrcFiles = fs.listStatus(srcDir).length;
+ fs.rename(srcDir, finalDir);
+ assertEquals(nSrcFiles, fs.listStatus(finalDir).length);
+ ContractTestUtils.assertPathDoesNotExist(fs, "not deleted after rename",
+ new Path(srcDir, "foo-" + 0));
+ ContractTestUtils.assertPathDoesNotExist(fs, "not deleted after rename",
+ new Path(srcDir, "foo-" + count / 2));
+ ContractTestUtils.assertPathDoesNotExist(fs, "not deleted after rename",
+ new Path(srcDir, "foo-" + (count - 1)));
+ ContractTestUtils.assertPathExists(fs, "not renamed to dest dir",
+ new Path(finalDir, "foo-" + 0));
+ ContractTestUtils.assertPathExists(fs, "not renamed to dest dir",
+ new Path(finalDir, "foo-" + count/2));
+ ContractTestUtils.assertPathExists(fs, "not renamed to dest dir",
+ new Path(finalDir, "foo-" + (count-1)));
+
+ ContractTestUtils.assertDeleted(fs, finalDir, true, false);
+ }
+
+ @Test
+ public void testOpenCreate() throws IOException {
+ Path dir = new Path("/tests3a");
+ ContractTestUtils.createAndVerifyFile(fs, dir, 1024);
+ ContractTestUtils.createAndVerifyFile(fs, dir, 5 * 1024 * 1024);
+ ContractTestUtils.createAndVerifyFile(fs, dir, 20 * 1024 * 1024);
+
+
+ /*
+ Enable to test the multipart upload
+ try {
+ ContractTestUtils.createAndVerifyFile(fs, dir,
+ (long)6 * 1024 * 1024 * 1024);
+ } catch (IOException e) {
+ fail(e.getMessage());
+ }
+ */
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/027d76fa/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADirectoryPerformance.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADirectoryPerformance.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADirectoryPerformance.java
new file mode 100644
index 0000000..b5f4eb3
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADirectoryPerformance.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.scale;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.Statistic;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+import static org.apache.hadoop.fs.s3a.Statistic.*;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.*;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
+
+/**
+ * Test the performance of listing files/directories.
+ */
+public class ITestS3ADirectoryPerformance extends S3AScaleTestBase {
+ private static final Logger LOG = LoggerFactory.getLogger(
+ ITestS3ADirectoryPerformance.class);
+
+ @Test
+ public void testListOperations() throws Throwable {
+ describe("Test recursive list operations");
+ final Path scaleTestDir = getTestPath();
+ final Path listDir = new Path(scaleTestDir, "lists");
+
+ // scale factor.
+ int scale = getConf().getInt(KEY_DIRECTORY_COUNT, DEFAULT_DIRECTORY_COUNT);
+ int width = scale;
+ int depth = scale;
+ int files = scale;
+ MetricDiff metadataRequests = new MetricDiff(fs, OBJECT_METADATA_REQUESTS);
+ MetricDiff listRequests = new MetricDiff(fs, OBJECT_LIST_REQUESTS);
+ MetricDiff listContinueRequests =
+ new MetricDiff(fs, OBJECT_CONTINUE_LIST_REQUESTS);
+ MetricDiff listStatusCalls = new MetricDiff(fs, INVOCATION_LIST_FILES);
+ MetricDiff getFileStatusCalls =
+ new MetricDiff(fs, INVOCATION_GET_FILE_STATUS);
+ NanoTimer createTimer = new NanoTimer();
+ TreeScanResults created =
+ createSubdirs(fs, listDir, depth, width, files, 0);
+ // add some empty directories
+ int emptyDepth = 1 * scale;
+ int emptyWidth = 3 * scale;
+
+ created.add(createSubdirs(fs, listDir, emptyDepth, emptyWidth, 0,
+ 0, "empty", "f-", ""));
+ createTimer.end("Time to create %s", created);
+ LOG.info("Time per operation: {}",
+ toHuman(createTimer.nanosPerOperation(created.totalCount())));
+ printThenReset(LOG,
+ metadataRequests,
+ listRequests,
+ listContinueRequests,
+ listStatusCalls,
+ getFileStatusCalls);
+
+ describe("Listing files via treewalk");
+ try {
+ // Scan the directory via an explicit tree walk.
+ // This is the baseline for any listing speedups.
+ NanoTimer treeWalkTimer = new NanoTimer();
+ TreeScanResults treewalkResults = treeWalk(fs, listDir);
+ treeWalkTimer.end("List status via treewalk of %s", created);
+
+ printThenReset(LOG,
+ metadataRequests,
+ listRequests,
+ listContinueRequests,
+ listStatusCalls,
+ getFileStatusCalls);
+ assertEquals("Files found in listFiles(recursive=true) " +
+ " created=" + created + " listed=" + treewalkResults,
+ created.getFileCount(), treewalkResults.getFileCount());
+
+ describe("Listing files via listFiles(recursive=true)");
+ // listFiles() does the recursion internally
+ NanoTimer listFilesRecursiveTimer = new NanoTimer();
+
+ TreeScanResults listFilesResults = new TreeScanResults(
+ fs.listFiles(listDir, true));
+
+ listFilesRecursiveTimer.end("listFiles(recursive=true) of %s", created);
+ assertEquals("Files found in listFiles(recursive=true) " +
+ " created=" + created + " listed=" + listFilesResults,
+ created.getFileCount(), listFilesResults.getFileCount());
+
+ // only two list operations should have taken place
+ print(LOG,
+ metadataRequests,
+ listRequests,
+ listContinueRequests,
+ listStatusCalls,
+ getFileStatusCalls);
+ assertEquals(listRequests.toString(), 2, listRequests.diff());
+ reset(metadataRequests,
+ listRequests,
+ listContinueRequests,
+ listStatusCalls,
+ getFileStatusCalls);
+
+
+ } finally {
+ describe("deletion");
+ // deletion at the end of the run
+ NanoTimer deleteTimer = new NanoTimer();
+ fs.delete(listDir, true);
+ deleteTimer.end("Deleting directory tree");
+ printThenReset(LOG,
+ metadataRequests,
+ listRequests,
+ listContinueRequests,
+ listStatusCalls,
+ getFileStatusCalls);
+ }
+ }
+
+ @Test
+ public void testTimeToStatEmptyDirectory() throws Throwable {
+ describe("Time to stat an empty directory");
+ Path path = new Path(getTestPath(), "empty");
+ fs.mkdirs(path);
+ timeToStatPath(path);
+ }
+
+ @Test
+ public void testTimeToStatNonEmptyDirectory() throws Throwable {
+ describe("Time to stat a non-empty directory");
+ Path path = new Path(getTestPath(), "dir");
+ fs.mkdirs(path);
+ touch(fs, new Path(path, "file"));
+ timeToStatPath(path);
+ }
+
+ @Test
+ public void testTimeToStatFile() throws Throwable {
+ describe("Time to stat a simple file");
+ Path path = new Path(getTestPath(), "file");
+ touch(fs, path);
+ timeToStatPath(path);
+ }
+
+ @Test
+ public void testTimeToStatRoot() throws Throwable {
+ describe("Time to stat the root path");
+ timeToStatPath(new Path("/"));
+ }
+
+ private void timeToStatPath(Path path) throws IOException {
+ describe("Timing getFileStatus(\"%s\")", path);
+ MetricDiff metadataRequests =
+ new MetricDiff(fs, Statistic.OBJECT_METADATA_REQUESTS);
+ MetricDiff listRequests =
+ new MetricDiff(fs, Statistic.OBJECT_LIST_REQUESTS);
+ long attempts = getOperationCount();
+ NanoTimer timer = new NanoTimer();
+ for (long l = 0; l < attempts; l++) {
+ fs.getFileStatus(path);
+ }
+ timer.end("Time to execute %d getFileStatusCalls", attempts);
+ LOG.info("Time per call: {}", toHuman(timer.nanosPerOperation(attempts)));
+ LOG.info("metadata: {}", metadataRequests);
+ LOG.info("metadata per operation {}", metadataRequests.diff() / attempts);
+ LOG.info("listObjects: {}", listRequests);
+ LOG.info("listObjects: per operation {}", listRequests.diff() / attempts);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/027d76fa/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AInputStreamPerformance.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AInputStreamPerformance.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AInputStreamPerformance.java
new file mode 100644
index 0000000..e2163c5
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AInputStreamPerformance.java
@@ -0,0 +1,534 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.scale;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.fs.s3a.S3AFileStatus;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.fs.s3a.S3AInputPolicy;
+import org.apache.hadoop.fs.s3a.S3AInputStream;
+import org.apache.hadoop.fs.s3a.S3AInstrumentation;
+import org.apache.hadoop.fs.s3a.S3ATestUtils;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.compress.CompressionCodec;
+import org.apache.hadoop.io.compress.CompressionCodecFactory;
+import org.apache.hadoop.util.LineReader;
+import org.junit.After;
+import org.junit.Assume;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.EOFException;
+import java.io.IOException;
+
+import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
+import static org.apache.hadoop.fs.s3a.Constants.*;
+
+/**
+ * Look at the performance of S3a operations.
+ */
+public class ITestS3AInputStreamPerformance extends S3AScaleTestBase {
+ private static final Logger LOG = LoggerFactory.getLogger(
+ ITestS3AInputStreamPerformance.class);
+
+ private S3AFileSystem s3aFS;
+ private Path testData;
+ private S3AFileStatus testDataStatus;
+ private FSDataInputStream in;
+ private S3AInstrumentation.InputStreamStatistics streamStatistics;
+ public static final int BLOCK_SIZE = 32 * 1024;
+ public static final int BIG_BLOCK_SIZE = 256 * 1024;
+
+ /** Tests only run if the there is a named test file that can be read. */
+ private boolean testDataAvailable = true;
+ private String assumptionMessage = "test file";
+
+ /**
+ * Open the FS and the test data. The input stream is always set up here.
+ * @throws IOException IO Problems.
+ */
+ @Before
+ public void openFS() throws IOException {
+ Configuration conf = getConf();
+ conf.setInt(SOCKET_SEND_BUFFER, 16 * 1024);
+ conf.setInt(SOCKET_RECV_BUFFER, 16 * 1024);
+ String testFile = conf.getTrimmed(KEY_CSVTEST_FILE, DEFAULT_CSVTEST_FILE);
+ if (testFile.isEmpty()) {
+ assumptionMessage = "Empty test property: " + KEY_CSVTEST_FILE;
+ testDataAvailable = false;
+ } else {
+ S3ATestUtils.useCSVDataEndpoint(conf);
+ testData = new Path(testFile);
+ Path path = this.testData;
+ bindS3aFS(path);
+ try {
+ testDataStatus = s3aFS.getFileStatus(this.testData);
+ } catch (IOException e) {
+ LOG.warn("Failed to read file {} specified in {}",
+ testFile, KEY_CSVTEST_FILE, e);
+ throw e;
+ }
+ }
+ }
+
+ private void bindS3aFS(Path path) throws IOException {
+ s3aFS = (S3AFileSystem) FileSystem.newInstance(path.toUri(), getConf());
+ }
+
+ /**
+ * Cleanup: close the stream, close the FS.
+ */
+ @After
+ public void cleanup() {
+ describe("cleanup");
+ IOUtils.closeStream(in);
+ IOUtils.closeStream(s3aFS);
+ }
+
+ /**
+ * Declare that the test requires the CSV test dataset.
+ */
+ private void requireCSVTestData() {
+ Assume.assumeTrue(assumptionMessage, testDataAvailable);
+ }
+
+ /**
+ * Open the test file with the read buffer specified in the setting.
+ * {@link #KEY_READ_BUFFER_SIZE}; use the {@code Normal} policy
+ * @return the stream, wrapping an S3a one
+ * @throws IOException IO problems
+ */
+ FSDataInputStream openTestFile() throws IOException {
+ return openTestFile(S3AInputPolicy.Normal, 0);
+ }
+
+ /**
+ * Open the test file with the read buffer specified in the setting
+ * {@link #KEY_READ_BUFFER_SIZE}.
+ * This includes the {@link #requireCSVTestData()} assumption; so
+ * if called before any FS op, will automatically skip the test
+ * if the CSV file is absent.
+ *
+ * @param inputPolicy input policy to use
+ * @param readahead readahead/buffer size
+ * @return the stream, wrapping an S3a one
+ * @throws IOException IO problems
+ */
+ FSDataInputStream openTestFile(S3AInputPolicy inputPolicy, long readahead)
+ throws IOException {
+ requireCSVTestData();
+ return openDataFile(s3aFS, this.testData, inputPolicy, readahead);
+ }
+
+ /**
+ * Open a test file with the read buffer specified in the setting
+ * {@link #KEY_READ_BUFFER_SIZE}.
+ *
+ * @param path path to open
+ * @param inputPolicy input policy to use
+ * @param readahead readahead/buffer size
+ * @return the stream, wrapping an S3a one
+ * @throws IOException IO problems
+ */
+ private FSDataInputStream openDataFile(S3AFileSystem fs,
+ Path path,
+ S3AInputPolicy inputPolicy,
+ long readahead) throws IOException {
+ int bufferSize = getConf().getInt(KEY_READ_BUFFER_SIZE,
+ DEFAULT_READ_BUFFER_SIZE);
+ S3AInputPolicy policy = fs.getInputPolicy();
+ fs.setInputPolicy(inputPolicy);
+ try {
+ FSDataInputStream stream = fs.open(path, bufferSize);
+ if (readahead >= 0) {
+ stream.setReadahead(readahead);
+ }
+ streamStatistics = getInputStreamStatistics(stream);
+ return stream;
+ } finally {
+ fs.setInputPolicy(policy);
+ }
+ }
+
+ /**
+ * Assert that the stream was only ever opened once.
+ */
+ protected void assertStreamOpenedExactlyOnce() {
+ assertOpenOperationCount(1);
+ }
+
+ /**
+ * Make an assertion count about the number of open operations.
+ * @param expected the expected number
+ */
+ private void assertOpenOperationCount(long expected) {
+ assertEquals("open operations in\n" + in,
+ expected, streamStatistics.openOperations);
+ }
+
+ /**
+ * Log how long an IOP took, by dividing the total time by the
+ * count of operations, printing in a human-readable form.
+ * @param operation operation being measured
+ * @param timer timing data
+ * @param count IOP count.
+ */
+ protected void logTimePerIOP(String operation,
+ NanoTimer timer,
+ long count) {
+ LOG.info("Time per {}: {} nS",
+ operation, toHuman(timer.duration() / count));
+ }
+
+ @Test
+ public void testTimeToOpenAndReadWholeFileBlocks() throws Throwable {
+ requireCSVTestData();
+ int blockSize = _1MB;
+ describe("Open the test file %s and read it in blocks of size %d",
+ testData, blockSize);
+ long len = testDataStatus.getLen();
+ in = openTestFile();
+ byte[] block = new byte[blockSize];
+ NanoTimer timer2 = new NanoTimer();
+ long count = 0;
+ // implicitly rounding down here
+ long blockCount = len / blockSize;
+ for (long i = 0; i < blockCount; i++) {
+ int offset = 0;
+ int remaining = blockSize;
+ NanoTimer blockTimer = new NanoTimer();
+ int reads = 0;
+ while (remaining > 0) {
+ int bytesRead = in.read(block, offset, remaining);
+ reads++;
+ if (bytesRead == 1) {
+ break;
+ }
+ remaining -= bytesRead;
+ offset += bytesRead;
+ count += bytesRead;
+ }
+ blockTimer.end("Reading block %d in %d reads", i, reads);
+ }
+ timer2.end("Time to read %d bytes in %d blocks", len, blockCount);
+ bandwidth(timer2, count);
+ logStreamStatistics();
+ }
+
+ @Test
+ public void testLazySeekEnabled() throws Throwable {
+ describe("Verify that seeks do not trigger any IO");
+ in = openTestFile();
+ long len = testDataStatus.getLen();
+ NanoTimer timer = new NanoTimer();
+ long blockCount = len / BLOCK_SIZE;
+ for (long i = 0; i < blockCount; i++) {
+ in.seek(in.getPos() + BLOCK_SIZE - 1);
+ }
+ in.seek(0);
+ blockCount++;
+ timer.end("Time to execute %d seeks", blockCount);
+ logTimePerIOP("seek()", timer, blockCount);
+ logStreamStatistics();
+ assertOpenOperationCount(0);
+ assertEquals("bytes read", 0, streamStatistics.bytesRead);
+ }
+
+ @Test
+ public void testReadaheadOutOfRange() throws Throwable {
+ try {
+ in = openTestFile();
+ in.setReadahead(-1L);
+ fail("Stream should have rejected the request "+ in);
+ } catch (IllegalArgumentException e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testReadWithNormalPolicy() throws Throwable {
+ describe("Read big blocks with a big readahead");
+ executeSeekReadSequence(BIG_BLOCK_SIZE, BIG_BLOCK_SIZE * 2,
+ S3AInputPolicy.Normal);
+ assertStreamOpenedExactlyOnce();
+ }
+
+ @Test
+ public void testDecompressionSequential128K() throws Throwable {
+ describe("Decompress with a 128K readahead");
+ executeDecompression(128 * 1024, S3AInputPolicy.Sequential);
+ assertStreamOpenedExactlyOnce();
+ }
+
+ /**
+ * Execute a decompression + line read with the given input policy.
+ * @param readahead byte readahead
+ * @param inputPolicy read policy
+ * @throws IOException IO Problems
+ */
+ private void executeDecompression(long readahead,
+ S3AInputPolicy inputPolicy) throws IOException {
+ CompressionCodecFactory factory
+ = new CompressionCodecFactory(getConf());
+ CompressionCodec codec = factory.getCodec(testData);
+ long bytesRead = 0;
+ int lines = 0;
+
+ FSDataInputStream objectIn = openTestFile(inputPolicy, readahead);
+ ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer();
+ try (LineReader lineReader = new LineReader(
+ codec.createInputStream(objectIn), getConf())) {
+ Text line = new Text();
+ int read;
+ while ((read = lineReader.readLine(line)) > 0) {
+ bytesRead += read;
+ lines++;
+ }
+ } catch (EOFException eof) {
+ // done
+ }
+ timer.end("Time to read %d lines [%d bytes expanded, %d raw]" +
+ " with readahead = %d",
+ lines,
+ bytesRead,
+ testDataStatus.getLen(),
+ readahead);
+ logTimePerIOP("line read", timer, lines);
+ logStreamStatistics();
+ }
+
+ private void logStreamStatistics() {
+ LOG.info(String.format("Stream Statistics%n{}"), streamStatistics);
+ }
+
+ /**
+ * Execute a seek+read sequence.
+ * @param blockSize block size for seeks
+ * @param readahead what the readahead value of the stream should be
+ * @throws IOException IO problems
+ */
+ protected void executeSeekReadSequence(long blockSize,
+ long readahead,
+ S3AInputPolicy policy) throws IOException {
+ in = openTestFile(policy, readahead);
+ long len = testDataStatus.getLen();
+ NanoTimer timer = new NanoTimer();
+ long blockCount = len / blockSize;
+ LOG.info("Reading {} blocks, readahead = {}",
+ blockCount, readahead);
+ for (long i = 0; i < blockCount; i++) {
+ in.seek(in.getPos() + blockSize - 1);
+ // this is the read
+ assertTrue(in.read() >= 0);
+ }
+ timer.end("Time to execute %d seeks of distance %d with readahead = %d",
+ blockCount,
+ blockSize,
+ readahead);
+ logTimePerIOP("seek(pos + " + blockCount+"); read()", timer, blockCount);
+ LOG.info("Effective bandwidth {} MB/S",
+ timer.bandwidthDescription(streamStatistics.bytesRead -
+ streamStatistics.bytesSkippedOnSeek));
+ logStreamStatistics();
+ }
+
+ public static final int _4K = 4 * 1024;
+ public static final int _8K = 8 * 1024;
+ public static final int _16K = 16 * 1024;
+ public static final int _32K = 32 * 1024;
+ public static final int _64K = 64 * 1024;
+ public static final int _128K = 128 * 1024;
+ public static final int _256K = 256 * 1024;
+ public static final int _1MB = 1024 * 1024;
+ public static final int _2MB = 2 * _1MB;
+ public static final int _10MB = _1MB * 10;
+ public static final int _5MB = _1MB * 5;
+
+ private static final int[][] RANDOM_IO_SEQUENCE = {
+ {_2MB, _128K},
+ {_128K, _128K},
+ {_5MB, _64K},
+ {_1MB, _1MB},
+ };
+
+ @Test
+ public void testRandomIORandomPolicy() throws Throwable {
+ executeRandomIO(S3AInputPolicy.Random, (long) RANDOM_IO_SEQUENCE.length);
+ assertEquals("streams aborted in " + streamStatistics,
+ 0, streamStatistics.aborted);
+ }
+
+ @Test
+ public void testRandomIONormalPolicy() throws Throwable {
+ long expectedOpenCount = RANDOM_IO_SEQUENCE.length;
+ executeRandomIO(S3AInputPolicy.Normal, expectedOpenCount);
+ assertEquals("streams aborted in " + streamStatistics,
+ 4, streamStatistics.aborted);
+ }
+
+ /**
+ * Execute the random IO {@code readFully(pos, bytes[])} sequence defined by
+ * {@link #RANDOM_IO_SEQUENCE}. The stream is closed afterwards; that's used
+ * in the timing too
+ * @param policy read policy
+ * @param expectedOpenCount expected number of stream openings
+ * @throws IOException IO problems
+ * @return the timer
+ */
+ private ContractTestUtils.NanoTimer executeRandomIO(S3AInputPolicy policy,
+ long expectedOpenCount)
+ throws IOException {
+ describe("Random IO with policy \"%s\"", policy);
+ byte[] buffer = new byte[_1MB];
+ long totalBytesRead = 0;
+
+ in = openTestFile(policy, 0);
+ ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer();
+ for (int[] action : RANDOM_IO_SEQUENCE) {
+ int position = action[0];
+ int range = action[1];
+ in.readFully(position, buffer, 0, range);
+ totalBytesRead += range;
+ }
+ int reads = RANDOM_IO_SEQUENCE.length;
+ timer.end("Time to execute %d reads of total size %d bytes",
+ reads,
+ totalBytesRead);
+ in.close();
+ assertOpenOperationCount(expectedOpenCount);
+ logTimePerIOP("byte read", timer, totalBytesRead);
+ LOG.info("Effective bandwidth {} MB/S",
+ timer.bandwidthDescription(streamStatistics.bytesRead -
+ streamStatistics.bytesSkippedOnSeek));
+ logStreamStatistics();
+ return timer;
+ }
+
+ S3AInputStream getS3aStream() {
+ return (S3AInputStream) in.getWrappedStream();
+ }
+
+ @Test
+ public void testRandomReadOverBuffer() throws Throwable {
+ describe("read over a buffer, making sure that the requests" +
+ " spans readahead ranges");
+ int datasetLen = _32K;
+ Path dataFile = new Path(getTestPath(), "testReadOverBuffer.bin");
+ byte[] sourceData = dataset(datasetLen, 0, 64);
+ // relies on the field 'fs' referring to the R/W FS
+ writeDataset(fs, dataFile, sourceData, datasetLen, _16K, true);
+ byte[] buffer = new byte[datasetLen];
+ int readahead = _8K;
+ int halfReadahead = _4K;
+ in = openDataFile(fs, dataFile, S3AInputPolicy.Random, readahead);
+
+ LOG.info("Starting initial reads");
+ S3AInputStream s3aStream = getS3aStream();
+ assertEquals(readahead, s3aStream.getReadahead());
+ byte[] oneByte = new byte[1];
+ assertEquals(1, in.read(0, oneByte, 0, 1));
+ // make some assertions about the current state
+ assertEquals("remaining in\n" + in,
+ readahead - 1, s3aStream.remainingInCurrentRequest());
+ assertEquals("range start in\n" + in,
+ 0, s3aStream.getContentRangeStart());
+ assertEquals("range finish in\n" + in,
+ readahead, s3aStream.getContentRangeFinish());
+
+ assertStreamOpenedExactlyOnce();
+
+ describe("Starting sequence of positioned read calls over\n%s", in);
+ NanoTimer readTimer = new NanoTimer();
+ int currentPos = halfReadahead;
+ int offset = currentPos;
+ int bytesRead = 0;
+ int readOps = 0;
+
+ // make multiple read() calls
+ while (bytesRead < halfReadahead) {
+ int length = buffer.length - offset;
+ int read = in.read(currentPos, buffer, offset, length);
+ bytesRead += read;
+ offset += read;
+ readOps++;
+ assertEquals("open operations on request #" + readOps
+ + " after reading " + bytesRead
+ + " current position in stream " + currentPos
+ + " in\n" + fs
+ + "\n " + in,
+ 1, streamStatistics.openOperations);
+ for (int i = currentPos; i < currentPos + read; i++) {
+ assertEquals("Wrong value from byte " + i,
+ sourceData[i], buffer[i]);
+ }
+ currentPos += read;
+ }
+ assertStreamOpenedExactlyOnce();
+ // assert at the end of the original block
+ assertEquals(readahead, currentPos);
+ readTimer.end("read %d in %d operations", bytesRead, readOps);
+ bandwidth(readTimer, bytesRead);
+ LOG.info("Time per byte(): {} nS",
+ toHuman(readTimer.nanosPerOperation(bytesRead)));
+ LOG.info("Time per read(): {} nS",
+ toHuman(readTimer.nanosPerOperation(readOps)));
+
+ describe("read last byte");
+ // read one more
+ int read = in.read(currentPos, buffer, bytesRead, 1);
+ assertTrue("-1 from last read", read >= 0);
+ assertOpenOperationCount(2);
+ assertEquals("Wrong value from read ", sourceData[currentPos],
+ (int) buffer[currentPos]);
+ currentPos++;
+
+
+ // now scan all the way to the end of the file, using single byte read()
+ // calls
+ describe("read() to EOF over \n%s", in);
+ long readCount = 0;
+ NanoTimer timer = new NanoTimer();
+ LOG.info("seeking");
+ in.seek(currentPos);
+ LOG.info("reading");
+ while(currentPos < datasetLen) {
+ int r = in.read();
+ assertTrue("Negative read() at position " + currentPos + " in\n" + in,
+ r >= 0);
+ buffer[currentPos] = (byte)r;
+ assertEquals("Wrong value from read from\n" + in,
+ sourceData[currentPos], r);
+ currentPos++;
+ readCount++;
+ }
+ timer.end("read %d bytes", readCount);
+ bandwidth(timer, readCount);
+ LOG.info("Time per read(): {} nS",
+ toHuman(timer.nanosPerOperation(readCount)));
+
+ assertEquals("last read in " + in, -1, in.read());
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org