You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by li...@apache.org on 2017/04/12 21:30:42 UTC
hadoop git commit: HADOOP-14255. S3A to delete unnecessary fake
directory objects in mkdirs(). Contributed by Mingliang Liu
Repository: hadoop
Updated Branches:
refs/heads/trunk c3375175d -> b053fdc54
HADOOP-14255. S3A to delete unnecessary fake directory objects in mkdirs(). Contributed by Mingliang Liu
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b053fdc5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b053fdc5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b053fdc5
Branch: refs/heads/trunk
Commit: b053fdc547ddbb6322674142a14010683006d123
Parents: c337517
Author: Mingliang Liu <li...@apache.org>
Authored: Thu Mar 30 13:03:34 2017 -0700
Committer: Mingliang Liu <li...@apache.org>
Committed: Wed Apr 12 14:30:34 2017 -0700
----------------------------------------------------------------------
.../fs/contract/AbstractContractMkdirTest.java | 42 ++++++++++++++++++++
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 3 +-
2 files changed, 43 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b053fdc5/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMkdirTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMkdirTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMkdirTest.java
index 427b0e9..71d2706 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMkdirTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMkdirTest.java
@@ -126,4 +126,46 @@ public abstract class AbstractContractMkdirTest extends AbstractFSContractTestBa
assertPathExists("check path existence without trailing slash failed",
path("testmkdir/b"));
}
+
+ @Test
+ public void testMkdirsPopulatingAllNonexistentAncestors() throws IOException {
+ describe("Verify mkdir will populate all its non-existent ancestors");
+ final FileSystem fs = getFileSystem();
+
+ final Path parent = path("testMkdirsPopulatingAllNonexistentAncestors");
+ assertTrue(fs.mkdirs(parent));
+ assertPathExists(parent + " should exist before making nested dir", parent);
+
+ Path nested = path(parent + "/a/b/c/d/e/f/g/h/i/j/k/L");
+ assertTrue(fs.mkdirs(nested));
+ while (nested != null && !nested.equals(parent) && !nested.isRoot()) {
+ assertPathExists(nested + " nested dir should exist", nested);
+ nested = nested.getParent();
+ }
+ }
+
+ @Test
+ public void testMkdirsDoesNotRemoveParentDirectories() throws IOException {
+ describe("Verify mkdir will make its parent existent");
+ final FileSystem fs = getFileSystem();
+
+ final Path parent = path("testMkdirsDoesNotRemoveParentDirectories");
+ assertTrue(fs.mkdirs(parent));
+
+ Path p = parent;
+ for (int i = 0; i < 10; i++) {
+ assertTrue(fs.mkdirs(p));
+ assertPathExists(p + " should exist after mkdir(" + p + ")", p);
+ p = path(p + "/dir-" + i);
+ }
+
+ // After mkdirs(sub-directory), its parent directory still exists
+ p = p.getParent();
+ while (p != null && !p.equals(parent) && !p.isRoot()) {
+ assertPathExists("Path " + p + " should exist", p);
+ assertIsDirectory(p);
+ p = p.getParent();
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b053fdc5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index b17281b..9eb5575 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -1525,8 +1525,6 @@ public class S3AFileSystem extends FileSystem {
* @throws IOException other IO problems
* @throws AmazonClientException on failures inside the AWS SDK
*/
- // TODO: If we have created an empty file at /foo/bar and we then call
- // mkdirs for /foo/bar/baz/roo what happens to the empty file /foo/bar/?
private boolean innerMkdirs(Path f, FsPermission permission)
throws IOException, FileAlreadyExistsException, AmazonClientException {
LOG.debug("Making directory: {}", f);
@@ -1561,6 +1559,7 @@ public class S3AFileSystem extends FileSystem {
String key = pathToKey(f);
createFakeDirectory(key);
+ deleteUnnecessaryFakeDirectories(f.getParent());
return true;
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org