You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/10/20 23:22:55 UTC
[31/50] [abbrv] kylin git commit: minor,
add cap for reducer number in RedistributeFlatHiveTableStep
minor, add cap for reducer number in RedistributeFlatHiveTableStep
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/111e7927
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/111e7927
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/111e7927
Branch: refs/heads/KYLIN-1971
Commit: 111e79275ecf4393d427a17f7f6654d74e2ec1a9
Parents: b4329a6
Author: shaofengshi <sh...@apache.org>
Authored: Tue Oct 18 18:41:02 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Oct 20 14:27:52 2016 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/source/hive/HiveMRInput.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/111e7927/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index f3fceb1..f536cbb 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -344,6 +344,7 @@ public class HiveMRInput implements IMRInput {
int numReducers = Math.round(rowCount / ((float) mapperInputRows));
numReducers = Math.max(1, numReducers);
+ numReducers = Math.min(numReducers, config.getHadoopJobMaxReducerNumber());
stepLogger.log("total input rows = " + rowCount);
stepLogger.log("expected input rows per mapper = " + mapperInputRows);