You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/11/26 10:43:49 UTC
kylin git commit: KYLIN-2470 loosen 0 mapper input bytes check for
partitioned hive table
Repository: kylin
Updated Branches:
refs/heads/master b40d0e06f -> d9e1d8cd9
KYLIN-2470 loosen 0 mapper input bytes check for partitioned hive table
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d9e1d8cd
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d9e1d8cd
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d9e1d8cd
Branch: refs/heads/master
Commit: d9e1d8cd9602936251628c72c63626e5a4f8cbf9
Parents: b40d0e0
Author: Li Yang <li...@apache.org>
Authored: Sun Nov 26 18:43:30 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Sun Nov 26 18:43:30 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/engine/mr/common/AbstractHadoopJob.java | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/d9e1d8cd/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index 1756251..74f8391 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -584,9 +584,12 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
for (InputSplit split : input.getSplits(job)) {
mapInputBytes += split.getLength();
}
+
+ // 0 input bytes is possible when the segment range hits no partition on a partitioned hive table (KYLIN-2470)
if (mapInputBytes == 0) {
- throw new IllegalArgumentException("Map input splits are 0 bytes, something is wrong!");
+ logger.warn("Map input splits are 0 bytes, something is wrong?");
}
+
double totalMapInputMB = (double) mapInputBytes / 1024 / 1024;
return totalMapInputMB;
}