You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2016/07/26 03:33:36 UTC
[17/55] [abbrv] kylin git commit: KYLIN-1656 set cap for max and min
reducer numbers
KYLIN-1656 set cap for max and min reducer numbers
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3ad49ddb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3ad49ddb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3ad49ddb
Branch: refs/heads/1.5.x-CDH5.7
Commit: 3ad49ddbbda5c7d88f7b47d2777ac97b2fa656cf
Parents: a08b77d
Author: shaofengshi <sh...@apache.org>
Authored: Mon Jul 11 14:32:04 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Jul 11 14:34:22 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/source/hive/CreateFlatHiveTableStep.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/3ad49ddb/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java b/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java
index 0297bfa..8a4aaa2 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java
@@ -62,7 +62,8 @@ public class CreateFlatHiveTableStep extends AbstractExecutable {
int mapperInputRows = config.getHadoopJobMapperInputRows();
int numReducers = Math.round(rowCount / ((float) mapperInputRows));
- numReducers = Math.max(1, numReducers);
+ numReducers = Math.max(numReducers, config.getHadoopJobMinReducerNumber());
+ numReducers = Math.min(numReducers, config.getHadoopJobMaxReducerNumber());
stepLogger.log("total input rows = " + rowCount);
stepLogger.log("expected input rows per mapper = " + mapperInputRows);