You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2018/05/17 18:09:29 UTC
[1/2] hadoop git commit: HADOOP-15469. S3A directory committer commit
job fails if _temporary directory created under dest. Contributed by Steve
Loughran.
Repository: hadoop
Updated Branches:
refs/heads/branch-3.1 f729faaab -> 170f1040d
refs/heads/trunk c0ec061e2 -> cc3600aab
HADOOP-15469. S3A directory committer commit job fails if _temporary directory created under dest.
Contributed by Steve Loughran.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/170f1040
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/170f1040
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/170f1040
Branch: refs/heads/branch-3.1
Commit: 170f1040d46f9b1a084e6637def91e9864446acc
Parents: f729faa
Author: Steve Loughran <st...@apache.org>
Authored: Thu May 17 19:08:05 2018 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu May 17 19:08:05 2018 +0100
----------------------------------------------------------------------
.../fs/s3a/commit/staging/DirectoryStagingCommitter.java | 7 ++-----
.../commit/staging/TestStagingDirectoryOutputCommitter.java | 8 +++-----
2 files changed, 5 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/170f1040/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/DirectoryStagingCommitter.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/DirectoryStagingCommitter.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/DirectoryStagingCommitter.java
index 03dfed2..3eda24f 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/DirectoryStagingCommitter.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/DirectoryStagingCommitter.java
@@ -93,11 +93,8 @@ public class DirectoryStagingCommitter extends StagingCommitter {
Configuration fsConf = fs.getConf();
switch (getConflictResolutionMode(context, fsConf)) {
case FAIL:
- // this was checked in setupJob, but this avoids some cases where
- // output was created while the job was processing
- if (fs.exists(outputPath)) {
- throw new PathExistsException(outputPath.toString(), E_DEST_EXISTS);
- }
+ // this was checked in setupJob; temporary files may have been
+ // created, so do not check again.
break;
case APPEND:
// do nothing
http://git-wip-us.apache.org/repos/asf/hadoop/blob/170f1040/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingDirectoryOutputCommitter.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingDirectoryOutputCommitter.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingDirectoryOutputCommitter.java
index 4f0189e..f5b8d1f 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingDirectoryOutputCommitter.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingDirectoryOutputCommitter.java
@@ -67,15 +67,14 @@ public class TestStagingDirectoryOutputCommitter
pathExists(mockS3, OUTPUT_PATH);
final DirectoryStagingCommitter committer = newJobCommitter();
+ // this should fail
intercept(PathExistsException.class,
InternalCommitterConstants.E_DEST_EXISTS,
"Should throw an exception because the path exists",
() -> committer.setupJob(getJob()));
- intercept(PathExistsException.class,
- InternalCommitterConstants.E_DEST_EXISTS,
- "Should throw an exception because the path exists",
- () -> committer.commitJob(getJob()));
+ // but there are no checks in job commit (HADOOP-15469)
+ committer.commitJob(getJob());
reset(mockS3);
pathDoesNotExist(mockS3, OUTPUT_PATH);
@@ -87,7 +86,6 @@ public class TestStagingDirectoryOutputCommitter
reset(mockS3);
pathDoesNotExist(mockS3, OUTPUT_PATH);
committer.commitJob(getJob());
- verifyExistenceChecked(mockS3, OUTPUT_PATH);
verifyCompletion(mockS3);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[2/2] hadoop git commit: HADOOP-15469. S3A directory committer commit
job fails if _temporary directory created under dest. Contributed by Steve
Loughran.
Posted by st...@apache.org.
HADOOP-15469. S3A directory committer commit job fails if _temporary directory created under dest.
Contributed by Steve Loughran.
(cherry picked from commit 170f1040d46f9b1a084e6637def91e9864446acc)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cc3600aa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cc3600aa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cc3600aa
Branch: refs/heads/trunk
Commit: cc3600aabdca6e8b14c9fe02fe54073bf4ef7685
Parents: c0ec061
Author: Steve Loughran <st...@apache.org>
Authored: Thu May 17 19:08:05 2018 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu May 17 19:08:55 2018 +0100
----------------------------------------------------------------------
.../fs/s3a/commit/staging/DirectoryStagingCommitter.java | 7 ++-----
.../commit/staging/TestStagingDirectoryOutputCommitter.java | 8 +++-----
2 files changed, 5 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc3600aa/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/DirectoryStagingCommitter.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/DirectoryStagingCommitter.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/DirectoryStagingCommitter.java
index 03dfed2..3eda24f 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/DirectoryStagingCommitter.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/DirectoryStagingCommitter.java
@@ -93,11 +93,8 @@ public class DirectoryStagingCommitter extends StagingCommitter {
Configuration fsConf = fs.getConf();
switch (getConflictResolutionMode(context, fsConf)) {
case FAIL:
- // this was checked in setupJob, but this avoids some cases where
- // output was created while the job was processing
- if (fs.exists(outputPath)) {
- throw new PathExistsException(outputPath.toString(), E_DEST_EXISTS);
- }
+ // this was checked in setupJob; temporary files may have been
+ // created, so do not check again.
break;
case APPEND:
// do nothing
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc3600aa/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingDirectoryOutputCommitter.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingDirectoryOutputCommitter.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingDirectoryOutputCommitter.java
index 4f0189e..f5b8d1f 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingDirectoryOutputCommitter.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingDirectoryOutputCommitter.java
@@ -67,15 +67,14 @@ public class TestStagingDirectoryOutputCommitter
pathExists(mockS3, OUTPUT_PATH);
final DirectoryStagingCommitter committer = newJobCommitter();
+ // this should fail
intercept(PathExistsException.class,
InternalCommitterConstants.E_DEST_EXISTS,
"Should throw an exception because the path exists",
() -> committer.setupJob(getJob()));
- intercept(PathExistsException.class,
- InternalCommitterConstants.E_DEST_EXISTS,
- "Should throw an exception because the path exists",
- () -> committer.commitJob(getJob()));
+ // but there are no checks in job commit (HADOOP-15469)
+ committer.commitJob(getJob());
reset(mockS3);
pathDoesNotExist(mockS3, OUTPUT_PATH);
@@ -87,7 +86,6 @@ public class TestStagingDirectoryOutputCommitter
reset(mockS3);
pathDoesNotExist(mockS3, OUTPUT_PATH);
committer.commitJob(getJob());
- verifyExistenceChecked(mockS3, OUTPUT_PATH);
verifyCompletion(mockS3);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org