You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ae...@apache.org on 2016/06/01 19:55:38 UTC
[37/39] hadoop git commit: HADOOP-13162. Consider reducing number of
getFileStatus calls in S3AFileSystem.mkdirs. (Rajesh Balamohan via stevel)
HADOOP-13162. Consider reducing number of getFileStatus calls in S3AFileSystem.mkdirs. (Rajesh Balamohan via stevel)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/58706110
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/58706110
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/58706110
Branch: refs/heads/HDFS-1312
Commit: 587061103097160d8aceb60dbef6958cafdd30ae
Parents: d749cf6
Author: Steve Loughran <st...@apache.org>
Authored: Wed Jun 1 14:17:18 2016 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Jun 1 14:18:20 2016 +0100
----------------------------------------------------------------------
.../fs/FileContextCreateMkdirBaseTest.java | 73 +++++++++++++++++++-
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 3 +
2 files changed, 73 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58706110/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextCreateMkdirBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextCreateMkdirBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextCreateMkdirBaseTest.java
index d91091f..c1de27a 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextCreateMkdirBaseTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextCreateMkdirBaseTest.java
@@ -20,14 +20,15 @@ package org.apache.hadoop.fs;
import java.io.IOException;
-import org.apache.hadoop.util.StringUtils;
import org.apache.log4j.Level;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import static org.apache.hadoop.fs.FileContextTestHelper.*;
-import org.apache.commons.logging.impl.Log4JLogger;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.assertIsDirectory;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.assertIsFile;
+
import org.apache.hadoop.test.GenericTestUtils;
/**
@@ -116,7 +117,73 @@ public abstract class FileContextCreateMkdirBaseTest {
fc.mkdir(f, FileContext.DEFAULT_PERM, true);
Assert.assertTrue(isDir(fc, f));
}
-
+
+ @Test
+ public void testMkdirsRecursiveWithExistingDir() throws IOException {
+ Path f = getTestRootPath(fc, "aDir/bDir/cDir");
+ fc.mkdir(f, FileContext.DEFAULT_PERM, true);
+ assertIsDirectory(fc.getFileStatus(f));
+ assertIsDirectory(fc.getFileStatus(f.getParent()));
+ assertIsDirectory(fc.getFileStatus(f.getParent().getParent()));
+ }
+
+ @Test
+ public void testMkdirRecursiveWithExistingFile() throws IOException {
+ Path f = getTestRootPath(fc, "NonExistant3/aDir");
+ fc.mkdir(f, FileContext.DEFAULT_PERM, true);
+ assertIsDirectory(fc.getFileStatus(f));
+ assertIsDirectory(fc.getFileStatus(f.getParent()));
+
+ // create a sample file
+ Path filePath = new Path(f.getParent(), "test.txt");
+ createFile(fc, filePath);
+ assertIsFile(filePath, fc.getFileStatus(filePath));
+
+ // try creating another folder which conflicts with filePath
+ Path dirPath = new Path(filePath, "bDir/cDir");
+ try {
+ fc.mkdir(dirPath, FileContext.DEFAULT_PERM, true);
+ Assert.fail("Mkdir for " + dirPath
+ + " should have failed as a file was present");
+ } catch(IOException e) {
+ // failed as expected
+ }
+ }
+
+ @Test
+ public void testWithRename() throws IOException, InterruptedException {
+ Path root = getTestRootPath(fc);
+ Path f = new Path(root, "d1/d2/d3");
+ fc.mkdir(f, FileContext.DEFAULT_PERM, true);
+ assertIsDirectory(fc.getFileStatus(new Path(root, "d1")));
+ assertIsDirectory(fc.getFileStatus(new Path(root, "d1/d2")));
+ assertIsDirectory(fc.getFileStatus(new Path(root, "d1/d2/d3")));
+
+ // create a sample file f.txt
+ Path fPath = new Path(root, "d1/d2/f.txt");
+ createFile(fc, fPath);
+ assertIsFile(fPath, fc.getFileStatus(fPath));
+ assertIsDirectory(fc.getFileStatus(new Path(root, "d1")));
+ assertIsDirectory(fc.getFileStatus(new Path(root, "d1/d2")));
+ assertIsDirectory(fc.getFileStatus(new Path(root, "d1/d2/d3")));
+
+ // create a sample file f2.txt
+ Path f2Path = new Path(getTestRootPath(fc), "d1/d2/d3/f2.txt");
+ createFile(fc, f2Path);
+ assertIsFile(fPath, fc.getFileStatus(f2Path));
+ assertIsDirectory(fc.getFileStatus(new Path(root, "d1")));
+ assertIsDirectory(fc.getFileStatus(new Path(root, "d1/d2")));
+ assertIsDirectory(fc.getFileStatus(new Path(root, "d1/d2/d3")));
+
+ //rename d1/d2/d3 d1/d4
+ fc.rename(new Path(root, "d1/d2/d3"), new Path(root, "d1/d4"));
+ assertIsDirectory(fc.getFileStatus(new Path(root, "d1")));
+ assertIsDirectory(fc.getFileStatus(new Path(root, "d1/d4")));
+ Path f2NewPath = new Path(root, "d1/d4/f2.txt");
+ assertIsFile(f2NewPath, fc.getFileStatus(f2NewPath));
+ }
+
+
///////////////////////
// Test Create
////////////////////////
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58706110/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 0eb720a..581518a 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -1088,6 +1088,9 @@ public class S3AFileSystem extends FileSystem {
do {
try {
FileStatus fileStatus = getFileStatus(fPart);
+ if (fileStatus.isDirectory()) {
+ break;
+ }
if (fileStatus.isFile()) {
throw new FileAlreadyExistsException(String.format(
"Can't make directory for path '%s' since it is a file.",
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org