You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2017/08/09 13:43:45 UTC
hadoop git commit: HADOOP-14505. Simplify mkdirs() after S3Guard
delete tracking change. Contributed by Sean Mackrory
Repository: hadoop
Updated Branches:
refs/heads/HADOOP-13345 884ae451c -> 85148a705
HADOOP-14505. Simplify mkdirs() after S3Guard delete tracking change.
Contributed by Sean Mackrory
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/85148a70
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/85148a70
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/85148a70
Branch: refs/heads/HADOOP-13345
Commit: 85148a70555c14f586ab078243fd30b2c207b466
Parents: 884ae45
Author: Steve Loughran <st...@apache.org>
Authored: Wed Aug 9 14:43:05 2017 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Aug 9 14:43:05 2017 +0100
----------------------------------------------------------------------
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 112 ++++++-------------
1 file changed, 35 insertions(+), 77 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85148a70/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 3279f1c..95fe94f 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -1670,41 +1670,6 @@ public class S3AFileSystem extends FileSystem {
}
}
- private enum DirectoryStatus {
- DOES_NOT_EXIST, EXISTS_AND_IS_DIRECTORY_ON_S3_ONLY,
- EXISTS_AND_IS_DIRECTORY_ON_METADATASTORE, EXISTS_AND_IS_FILE
- }
-
- private DirectoryStatus checkPathForDirectory(Path path) throws
- IOException {
- try {
- if (path.isRoot()) {
- return DirectoryStatus.EXISTS_AND_IS_DIRECTORY_ON_METADATASTORE;
- }
- String key = pathToKey(path);
-
- // Check MetadataStore, if any.
- FileStatus status = null;
- PathMetadata pm = metadataStore.get(path, false);
- if (pm != null) {
- if (pm.isDeleted()) {
- return DirectoryStatus.DOES_NOT_EXIST;
- }
- status = pm.getFileStatus();
- if (status != null && status.isDirectory()) {
- return DirectoryStatus.EXISTS_AND_IS_DIRECTORY_ON_METADATASTORE;
- }
- }
- status = s3GetFileStatus(path, key, null);
- if (status.isDirectory()) {
- return DirectoryStatus.EXISTS_AND_IS_DIRECTORY_ON_S3_ONLY;
- }
- } catch (FileNotFoundException e) {
- return DirectoryStatus.DOES_NOT_EXIST;
- }
- return DirectoryStatus.EXISTS_AND_IS_FILE;
- }
-
/**
*
* Make the given path and all non-existent parents into
@@ -1719,65 +1684,58 @@ public class S3AFileSystem extends FileSystem {
*/
private boolean innerMkdirs(Path p, FsPermission permission)
throws IOException, FileAlreadyExistsException, AmazonClientException {
- boolean createOnS3 = false;
Path f = qualify(p);
LOG.debug("Making directory: {}", f);
incrementStatistic(INVOCATION_MKDIRS);
+ FileStatus fileStatus;
List<Path> metadataStoreDirs = null;
if (hasMetadataStore()) {
metadataStoreDirs = new ArrayList<>();
}
- DirectoryStatus status = checkPathForDirectory(f);
- if (status == DirectoryStatus.DOES_NOT_EXIST) {
- createOnS3 = true;
- if (metadataStoreDirs != null) {
- metadataStoreDirs.add(f);
+ try {
+ fileStatus = getFileStatus(f);
+
+ if (fileStatus.isDirectory()) {
+ return true;
+ } else {
+ throw new FileAlreadyExistsException("Path is a file: " + f);
}
- } else if (status == DirectoryStatus.EXISTS_AND_IS_DIRECTORY_ON_S3_ONLY) {
+ } catch (FileNotFoundException e) {
+ // Walk path to root, ensuring closest ancestor is a directory, not file
+ Path fPart = f.getParent();
if (metadataStoreDirs != null) {
metadataStoreDirs.add(f);
}
- } else if (status == DirectoryStatus
- .EXISTS_AND_IS_DIRECTORY_ON_METADATASTORE) {
- return true;
- } else if (status == DirectoryStatus.EXISTS_AND_IS_FILE) {
- throw new FileAlreadyExistsException("Path is a file: " + f);
- }
-
- // Walk path to root, ensuring closest ancestor is a directory, not file
- Path fPart = f.getParent();
- do {
- status = checkPathForDirectory(fPart);
- // The fake directory on S3 may not be visible immediately, but
- // only the leaf node has a fake directory on S3 so we can treat both
- // cases the same and just create a metadata store entry.
- if (status == DirectoryStatus.DOES_NOT_EXIST || status ==
- DirectoryStatus.EXISTS_AND_IS_DIRECTORY_ON_S3_ONLY) {
- if (metadataStoreDirs != null) {
- metadataStoreDirs.add(fPart);
+ while (fPart != null) {
+ try {
+ fileStatus = getFileStatus(fPart);
+ if (fileStatus.isDirectory()) {
+ break;
+ }
+ if (fileStatus.isFile()) {
+ throw new FileAlreadyExistsException(String.format(
+ "Can't make directory for path '%s' since it is a file.",
+ fPart));
+ }
+ } catch (FileNotFoundException fnfe) {
+ instrumentation.errorIgnored();
+ // We create all missing directories in MetadataStore; it does not
+ // infer directories exist by prefix like S3.
+ if (metadataStoreDirs != null) {
+ metadataStoreDirs.add(fPart);
+ }
}
- } else if (status == DirectoryStatus
- .EXISTS_AND_IS_DIRECTORY_ON_METADATASTORE) {
- // do nothing - just break out of the loop, make whatever child
- // directories are needed on the metadata store, and return the
- // result.
- break;
- } else if (status == DirectoryStatus.EXISTS_AND_IS_FILE) {
- throw new FileAlreadyExistsException("Path is a file: " + f);
+ fPart = fPart.getParent();
}
- fPart = fPart.getParent();
- } while (fPart != null);
-
- if (createOnS3) {
String key = pathToKey(f);
createFakeDirectory(key);
+ S3Guard.makeDirsOrdered(metadataStore, metadataStoreDirs, username, true);
+ // this is complicated because getParent(a/b/c/) returns a/b/c, but
+ // we want a/b. See HADOOP-14428 for more details.
+ deleteUnnecessaryFakeDirectories(new Path(f.toString()).getParent());
+ return true;
}
- S3Guard.makeDirsOrdered(metadataStore, metadataStoreDirs, username, true);
- // this is complicated because getParent(a/b/c/) returns a/b/c, but
- // we want a/b. See HADOOP-14428 for more details.
- deleteUnnecessaryFakeDirectories(new Path(f.toString()).getParent());
- return true;
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org