You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cn...@apache.org on 2016/08/18 20:27:34 UTC
[32/50] [abbrv] hadoop git commit: MAPREDUCE-6751. Add debug log
message when splitting is not possible due to unsplittable compression.
(Peter Vary via rchiang)
MAPREDUCE-6751. Add debug log message when splitting is not possible due to unsplittable compression. (Peter Vary via rchiang)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6c154abd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6c154abd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6c154abd
Branch: refs/heads/HADOOP-13345
Commit: 6c154abd33279475315b5f7f78dc47f1b0aa7028
Parents: b047bc7
Author: Ray Chiang <rc...@apache.org>
Authored: Tue Aug 16 12:13:22 2016 -0700
Committer: Ray Chiang <rc...@apache.org>
Committed: Tue Aug 16 12:13:22 2016 -0700
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/mapred/FileInputFormat.java | 7 +++++++
.../hadoop/mapreduce/lib/input/CombineFileInputFormat.java | 4 ++++
.../apache/hadoop/mapreduce/lib/input/FileInputFormat.java | 7 +++++++
3 files changed, 18 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c154abd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java
index 2c58ebe..5803d60 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java
@@ -369,6 +369,13 @@ public abstract class FileInputFormat<K, V> implements InputFormat<K, V> {
splitHosts[0], splitHosts[1]));
}
} else {
+ if (LOG.isDebugEnabled()) {
+ // Log only if the file is big enough to be splitted
+ if (length > Math.min(file.getBlockSize(), minSize)) {
+ LOG.debug("File is not splittable so no parallelization "
+ + "is possible: " + file.getPath());
+ }
+ }
String[][] splitHosts = getSplitHostsAndCachedHosts(blkLocations,0,length,clusterMap);
splits.add(makeSplit(path, 0, length, splitHosts[0], splitHosts[1]));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c154abd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileInputFormat.java
index b2b7656..8f9699e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileInputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileInputFormat.java
@@ -600,6 +600,10 @@ public abstract class CombineFileInputFormat<K, V>
if (!isSplitable) {
// if the file is not splitable, just create the one block with
// full file length
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("File is not splittable so no parallelization "
+ + "is possible: " + stat.getPath());
+ }
blocks = new OneBlockInfo[1];
fileSize = stat.getLen();
blocks[0] = new OneBlockInfo(stat.getPath(), 0, fileSize,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c154abd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java
index 0c5ede9..7ec882f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java
@@ -422,6 +422,13 @@ public abstract class FileInputFormat<K, V> extends InputFormat<K, V> {
blkLocations[blkIndex].getCachedHosts()));
}
} else { // not splitable
+ if (LOG.isDebugEnabled()) {
+ // Log only if the file is big enough to be splitted
+ if (length > Math.min(file.getBlockSize(), minSize)) {
+ LOG.debug("File is not splittable so no parallelization "
+ + "is possible: " + file.getPath());
+ }
+ }
splits.add(makeSplit(path, 0, length, blkLocations[0].getHosts(),
blkLocations[0].getCachedHosts()));
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org