You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xy...@apache.org on 2017/06/08 20:06:12 UTC
[30/50] [abbrv] hadoop git commit: HADOOP-14428. s3a: mkdir appears
to be broken. Contributed by Mingliang Liu
HADOOP-14428. s3a: mkdir appears to be broken. Contributed by Mingliang Liu
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ce634881
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ce634881
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ce634881
Branch: refs/heads/HDFS-7240
Commit: ce634881ced7ff14118a7789cb70ff6428710e00
Parents: 323f8bb
Author: Mingliang Liu <li...@apache.org>
Authored: Wed May 24 14:44:27 2017 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Jun 8 10:44:51 2017 -0700
----------------------------------------------------------------------
.../fs/contract/AbstractContractMkdirTest.java | 31 ++++++++++++--------
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 5 +++-
2 files changed, 23 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce634881/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMkdirTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMkdirTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMkdirTest.java
index 71d2706..c5a546d 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMkdirTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMkdirTest.java
@@ -113,18 +113,25 @@ public abstract class AbstractContractMkdirTest extends AbstractFSContractTestBa
describe("verify mkdir slash handling");
FileSystem fs = getFileSystem();
- // No trailing slash
- assertTrue(fs.mkdirs(path("testmkdir/a")));
- assertPathExists("mkdir without trailing slash failed",
- path("testmkdir/a"));
-
- // With trailing slash
- assertTrue(fs.mkdirs(path("testmkdir/b/")));
- assertPathExists("mkdir with trailing slash failed", path("testmkdir/b/"));
-
- // Mismatched slashes
- assertPathExists("check path existence without trailing slash failed",
- path("testmkdir/b"));
+ final Path[] paths = new Path[] {
+ path("testMkdirSlashHandling/a"), // w/o trailing slash
+ path("testMkdirSlashHandling/b/"), // w/ trailing slash
+ // unqualified w/o trailing slash
+ new Path(getContract().getTestPath() + "/testMkdirSlashHandling/c"),
+ // unqualified w/ trailing slash
+ new Path(getContract().getTestPath() + "/testMkdirSlashHandling/d/"),
+ // unqualified w/ multiple trailing slashes
+ new Path(getContract().getTestPath() + "/testMkdirSlashHandling/e///")
+ };
+ for (Path path : paths) {
+ assertTrue(fs.mkdirs(path));
+ assertPathExists(path + " does not exist after mkdirs", path);
+ assertIsDirectory(path);
+ if (path.toString().endsWith("/")) {
+ String s = path.toString().substring(0, path.toString().length() - 1);
+ assertIsDirectory(new Path(s));
+ }
+ }
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce634881/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 25f2671..872dd5f 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -1587,7 +1587,9 @@ public class S3AFileSystem extends FileSystem {
String key = pathToKey(f);
createFakeDirectory(key);
- deleteUnnecessaryFakeDirectories(f.getParent());
+ // this is complicated because getParent(a/b/c/) returns a/b/c, but
+ // we want a/b. See HADOOP-14428 for more details.
+ deleteUnnecessaryFakeDirectories(new Path(f.toString()).getParent());
return true;
}
}
@@ -1971,6 +1973,7 @@ public class S3AFileSystem extends FileSystem {
while (!path.isRoot()) {
String key = pathToKey(path);
key = (key.endsWith("/")) ? key : (key + "/");
+ LOG.trace("To delete unnecessary fake directory {} for {}", key, path);
keysToRemove.add(new DeleteObjectsRequest.KeyVersion(key));
path = path.getParent();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org