You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2023/02/22 19:34:36 UTC
[hadoop] branch branch-3.3 updated: HADOOP-18582. Addendum: Skip unnecessary cleanup logic in DistCp. (#5409)
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new 84e999b35c1 HADOOP-18582. Addendum: Skip unnecessary cleanup logic in DistCp. (#5409)
84e999b35c1 is described below
commit 84e999b35c12126daaf022537f4d44caef61f145
Author: Ayush Saxena <ay...@apache.org>
AuthorDate: Thu Feb 23 00:59:41 2023 +0530
HADOOP-18582. Addendum: Skip unnecessary cleanup logic in DistCp. (#5409)
Followup to the original HADOOP-18582.
Temporary path cleanup is re-enabled for -append jobs
as these will create temporary files when creating or overwriting files.
Contributed by Ayush Saxena
---
.../java/org/apache/hadoop/tools/mapred/CopyCommitter.java | 5 +----
.../org/apache/hadoop/tools/mapred/TestCopyCommitter.java | 12 +++++-------
2 files changed, 6 insertions(+), 11 deletions(-)
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java
index e5c74094e90..1cafb23febe 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java
@@ -153,10 +153,7 @@ public class CopyCommitter extends FileOutputCommitter {
final boolean directWrite = conf.getBoolean(
DistCpOptionSwitch.DIRECT_WRITE.getConfigLabel(), false);
- final boolean append = conf.getBoolean(
- DistCpOptionSwitch.APPEND.getConfigLabel(), false);
- final boolean useTempTarget = !append && !directWrite;
- if (!useTempTarget) {
+ if (directWrite) {
return;
}
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java
index f2dd246db5a..6a537dc6e7d 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java
@@ -586,13 +586,11 @@ public class TestCopyCommitter {
@Test
public void testCommitWithCleanupTempFiles() throws IOException {
- testCommitWithCleanup(true, false);
- testCommitWithCleanup(false, true);
- testCommitWithCleanup(true, true);
- testCommitWithCleanup(false, false);
+ testCommitWithCleanup(true);
+ testCommitWithCleanup(false);
}
- private void testCommitWithCleanup(boolean append, boolean directWrite)throws IOException {
+ private void testCommitWithCleanup(boolean directWrite) throws IOException {
TaskAttemptContext taskAttemptContext = getTaskAttemptContext(config);
JobID jobID = taskAttemptContext.getTaskAttemptID().getJobID();
JobContext jobContext = new JobContextImpl(
@@ -611,7 +609,7 @@ public class TestCopyCommitter {
DistCpOptions options = new DistCpOptions.Builder(
Collections.singletonList(new Path(sourceBase)),
new Path("/out"))
- .withAppend(append)
+ .withAppend(true)
.withSyncFolder(true)
.withDirectWrite(directWrite)
.build();
@@ -631,7 +629,7 @@ public class TestCopyCommitter {
null, taskAttemptContext);
committer.commitJob(jobContext);
- if (append || directWrite) {
+ if (directWrite) {
ContractTestUtils.assertPathExists(fs, "Temp files should not be cleanup with append or direct option",
tempFilePath);
} else {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org