You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2021/01/27 19:09:26 UTC
[hadoop] branch branch-3.3 updated: MAPREDUCE-7317. Add latency
information in FileOutputCommitter.mergePaths. (#2624)
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new ebdaced MAPREDUCE-7317. Add latency information in FileOutputCommitter.mergePaths. (#2624)
ebdaced is described below
commit ebdacedc83580f6ec92a03129328e13718c12cad
Author: Jungtaek Lim <ka...@gmail.com>
AuthorDate: Thu Jan 28 04:08:08 2021 +0900
MAPREDUCE-7317. Add latency information in FileOutputCommitter.mergePaths. (#2624)
Contributed by Jungtaek Lim.
Change-Id: Iaff2f55e5378c22ce8a92ae776f5aba3f0fc304e
---
.../mapreduce/lib/output/FileOutputCommitter.java | 62 +++++++++++-----------
1 file changed, 32 insertions(+), 30 deletions(-)
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputCommitter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputCommitter.java
index e8f9ec7..2973fb0 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputCommitter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputCommitter.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.DurationInfo;
import org.apache.hadoop.util.Progressable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -454,43 +455,44 @@ public class FileOutputCommitter extends PathOutputCommitter {
*/
private void mergePaths(FileSystem fs, final FileStatus from,
final Path to, JobContext context) throws IOException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Merging data from " + from + " to " + to);
- }
- reportProgress(context);
- FileStatus toStat;
- try {
- toStat = fs.getFileStatus(to);
- } catch (FileNotFoundException fnfe) {
- toStat = null;
- }
-
- if (from.isFile()) {
- if (toStat != null) {
- if (!fs.delete(to, true)) {
- throw new IOException("Failed to delete " + to);
- }
+ try (DurationInfo d = new DurationInfo(LOG,
+ false,
+ "Merging data from %s to %s", from, to)) {
+ reportProgress(context);
+ FileStatus toStat;
+ try {
+ toStat = fs.getFileStatus(to);
+ } catch (FileNotFoundException fnfe) {
+ toStat = null;
}
- if (!fs.rename(from.getPath(), to)) {
- throw new IOException("Failed to rename " + from + " to " + to);
- }
- } else if (from.isDirectory()) {
- if (toStat != null) {
- if (!toStat.isDirectory()) {
+ if (from.isFile()) {
+ if (toStat != null) {
if (!fs.delete(to, true)) {
throw new IOException("Failed to delete " + to);
}
- renameOrMerge(fs, from, to, context);
- } else {
- //It is a directory so merge everything in the directories
- for (FileStatus subFrom : fs.listStatus(from.getPath())) {
- Path subTo = new Path(to, subFrom.getPath().getName());
- mergePaths(fs, subFrom, subTo, context);
+ }
+
+ if (!fs.rename(from.getPath(), to)) {
+ throw new IOException("Failed to rename " + from + " to " + to);
+ }
+ } else if (from.isDirectory()) {
+ if (toStat != null) {
+ if (!toStat.isDirectory()) {
+ if (!fs.delete(to, true)) {
+ throw new IOException("Failed to delete " + to);
+ }
+ renameOrMerge(fs, from, to, context);
+ } else {
+ //It is a directory so merge everything in the directories
+ for (FileStatus subFrom : fs.listStatus(from.getPath())) {
+ Path subTo = new Path(to, subFrom.getPath().getName());
+ mergePaths(fs, subFrom, subTo, context);
+ }
}
+ } else {
+ renameOrMerge(fs, from, to, context);
}
- } else {
- renameOrMerge(fs, from, to, context);
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org