You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by wa...@apache.org on 2015/09/18 19:19:42 UTC
hadoop git commit: MAPREDUCE-6478. Add an option to skip cleanupJob
stage or ignore cleanup failure during commitJob. (Junping Du via wangda)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 4a657e932 -> 6041fc75d
MAPREDUCE-6478. Add an option to skip cleanupJob stage or ignore cleanup failure during commitJob. (Junping Du via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6041fc75
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6041fc75
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6041fc75
Branch: refs/heads/branch-2
Commit: 6041fc75d85d9646097b3169b83e54681df660e1
Parents: 4a657e9
Author: Wangda Tan <wa...@apache.org>
Authored: Fri Sep 18 10:19:32 2015 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Fri Sep 18 10:19:32 2015 -0700
----------------------------------------------------------------------
.../lib/output/FileOutputCommitter.java | 52 ++++++++++++++++++--
hadoop-yarn-project/CHANGES.txt | 3 ++
2 files changed, 52 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6041fc75/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputCommitter.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputCommitter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputCommitter.java
index 6e5d0a1..b749a34 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputCommitter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputCommitter.java
@@ -64,9 +64,23 @@ public class FileOutputCommitter extends OutputCommitter {
public static final String FILEOUTPUTCOMMITTER_ALGORITHM_VERSION =
"mapreduce.fileoutputcommitter.algorithm.version";
public static final int FILEOUTPUTCOMMITTER_ALGORITHM_VERSION_DEFAULT = 1;
+ // Skip cleanup _temporary folders under job's output directory
+ public static final String FILEOUTPUTCOMMITTER_CLEANUP_SKIPPED =
+ "mapreduce.fileoutputcommitter.cleanup.skipped";
+ public static final boolean
+ FILEOUTPUTCOMMITTER_CLEANUP_SKIPPED_DEFAULT = false;
+
+ // Ignore exceptions in cleanup _temporary folder under job's output directory
+ public static final String FILEOUTPUTCOMMITTER_CLEANUP_FAILURES_IGNORED =
+ "mapreduce.fileoutputcommitter.cleanup-failures.ignored";
+ public static final boolean
+ FILEOUTPUTCOMMITTER_CLEANUP_FAILURES_IGNORED_DEFAULT = false;
+
private Path outputPath = null;
private Path workPath = null;
private final int algorithmVersion;
+ private final boolean skipCleanup;
+ private final boolean ignoreCleanupFailures;
/**
* Create a file output committer
@@ -101,6 +115,21 @@ public class FileOutputCommitter extends OutputCommitter {
if (algorithmVersion != 1 && algorithmVersion != 2) {
throw new IOException("Only 1 or 2 algorithm version is supported");
}
+
+ // if skip cleanup
+ skipCleanup = conf.getBoolean(
+ FILEOUTPUTCOMMITTER_CLEANUP_SKIPPED,
+ FILEOUTPUTCOMMITTER_CLEANUP_SKIPPED_DEFAULT);
+
+ // if ignore failures in cleanup
+ ignoreCleanupFailures = conf.getBoolean(
+ FILEOUTPUTCOMMITTER_CLEANUP_FAILURES_IGNORED,
+ FILEOUTPUTCOMMITTER_CLEANUP_FAILURES_IGNORED_DEFAULT);
+
+ LOG.info("FileOutputCommitter skip cleanup _temporary folders under " +
+ "output directory:" + skipCleanup + ", ignore cleanup failures: " +
+ ignoreCleanupFailures);
+
if (outputPath != null) {
FileSystem fs = outputPath.getFileSystem(context.getConfiguration());
this.outputPath = fs.makeQualified(outputPath);
@@ -327,8 +356,25 @@ public class FileOutputCommitter extends OutputCommitter {
}
}
- // delete the _temporary folder and create a _done file in the o/p folder
- cleanupJob(context);
+ if (skipCleanup) {
+ LOG.info("Skip cleanup the _temporary folders under job's output " +
+ "directory in commitJob.");
+ } else {
+ // delete the _temporary folder and create a _done file in the o/p
+ // folder
+ try {
+ cleanupJob(context);
+ } catch (IOException e) {
+ if (ignoreCleanupFailures) {
+ // swallow exceptions in cleanup as user configure to make sure
+ // commitJob could be success even when cleanup get failure.
+ LOG.error("Error in cleanup job, manually cleanup is needed.", e);
+ } else {
+ // throw back exception to fail commitJob.
+ throw e;
+ }
+ }
+ }
// True if the job requires output.dir marked on successful job.
// Note that by default it is set to true.
if (context.getConfiguration().getBoolean(SUCCESSFUL_JOB_OUTPUT_DIR_MARKER, true)) {
@@ -600,4 +646,4 @@ public class FileOutputCommitter extends OutputCommitter {
LOG.warn("Output Path is null in recoverTask()");
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6041fc75/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 0d91e8d..d06b5e0 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -399,6 +399,9 @@ Release 2.8.0 - UNRELEASED
YARN-2597. MiniYARNCluster should propagate reason for AHS not starting.
(stevel)
+ MAPREDUCE-6478. Add an option to skip cleanupJob stage or ignore cleanup
+ failure during commitJob. (Junping Du via wangda)
+
OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not