You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by go...@apache.org on 2016/07/11 23:23:38 UTC
hive git commit: HIVE-14200: Tez: disable auto-reducer parallelism
when reducer-count * min.partition.factor < 1.0 (Gopal V,
reviewed by Gunther Hagleitner)
Repository: hive
Updated Branches:
refs/heads/master c790391cc -> a61c35184
HIVE-14200: Tez: disable auto-reducer parallelism when reducer-count * min.partition.factor < 1.0 (Gopal V, reviewed by Gunther Hagleitner)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a61c3518
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a61c3518
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a61c3518
Branch: refs/heads/master
Commit: a61c351848ba358b3834d9ace934f01a60d8a021
Parents: c790391
Author: Gopal V <go...@apache.org>
Authored: Mon Jul 11 16:22:59 2016 -0700
Committer: Gopal V <go...@apache.org>
Committed: Mon Jul 11 16:22:59 2016 -0700
----------------------------------------------------------------------
.../hadoop/hive/ql/parse/GenTezUtils.java | 28 +++++++++++++-------
1 file changed, 18 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/a61c3518/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
index d5a2eca..32bf24d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
@@ -107,23 +107,31 @@ public class GenTezUtils {
reduceWork.setNumReduceTasks(reduceSink.getConf().getNumReducers());
if (isAutoReduceParallelism && reduceSink.getConf().getReducerTraits().contains(AUTOPARALLEL)) {
- reduceWork.setAutoReduceParallelism(true);
// configured limit for reducers
- int maxReducers = context.conf.getIntVar(HiveConf.ConfVars.MAXREDUCERS);
+ final int maxReducers = context.conf.getIntVar(HiveConf.ConfVars.MAXREDUCERS);
+ // estimated number of reducers
+ final int nReducers = reduceSink.getConf().getNumReducers();
// min we allow tez to pick
- int minPartition = Math.max(1, (int) (reduceSink.getConf().getNumReducers()
- * minPartitionFactor));
+ int minPartition = Math.max(1, (int) (nReducers * minPartitionFactor));
minPartition = (minPartition > maxReducers) ? maxReducers : minPartition;
// max we allow tez to pick
- int maxPartition = (int) (reduceSink.getConf().getNumReducers() * maxPartitionFactor);
- maxPartition = Math.max(1, (maxPartition > maxReducers) ? maxReducers :
- maxPartition);
-
- reduceWork.setMinReduceTasks(minPartition);
- reduceWork.setMaxReduceTasks(maxPartition);
+ int maxPartition = Math.max(1, (int) (nReducers * maxPartitionFactor));
+ maxPartition = (maxPartition > maxReducers) ? maxReducers : maxPartition;
+
+ // reduce only if the parameters are significant
+ if (minPartition < maxPartition &&
+ nReducers * minPartitionFactor >= 1.0) {
+ reduceWork.setAutoReduceParallelism(true);
+
+ reduceWork.setMinReduceTasks(minPartition);
+ reduceWork.setMaxReduceTasks(maxPartition);
+ } else if (nReducers < maxPartition) {
+ // the max is good, the min is too low
+ reduceWork.setNumReduceTasks(maxPartition);
+ }
}
setupReduceSink(context, reduceWork, reduceSink);