You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2016/01/07 17:28:06 UTC
[2/2] hive git commit: HIVE-12664 : Bug in reduce deduplication
optimization causing ArrayOutOfBoundException (Johan Gustavsson via Ashutosh
Chauhan)
HIVE-12664 : Bug in reduce deduplication optimization causing ArrayOutOfBoundException (Johan Gustavsson via Ashutosh Chauhan)
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a6debb95
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a6debb95
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a6debb95
Branch: refs/heads/master
Commit: a6debb95da0365415a757d3522e9e1dbfdfe0dd9
Parents: cb17456
Author: Johan Gustavsson <jo...@treasure-data.com>
Authored: Mon Jan 4 18:01:00 2016 -0800
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Thu Jan 7 08:27:47 2016 -0800
----------------------------------------------------------------------
.../correlation/ReduceSinkDeDuplication.java | 39 ++++++++++----------
1 file changed, 20 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/a6debb95/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java
index 8da9ad5..1746dd3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java
@@ -186,22 +186,24 @@ public class ReduceSinkDeDuplication extends Transform {
List<Operator<?>> parents = pJoin.getParentOperators();
ReduceSinkOperator[] pRSs = parents.toArray(new ReduceSinkOperator[parents.size()]);
ReduceSinkDesc cRSc = cRS.getConf();
- ReduceSinkDesc pRS0c = pRSs[0].getConf();
- if (cRSc.getKeyCols().size() < pRS0c.getKeyCols().size()) {
- return false;
- }
- if (cRSc.getPartitionCols().size() != pRS0c.getPartitionCols().size()) {
- return false;
- }
- Integer moveReducerNumTo = checkNumReducer(cRSc.getNumReducers(), pRS0c.getNumReducers());
- if (moveReducerNumTo == null ||
- moveReducerNumTo > 0 && cRSc.getNumReducers() < minReducer) {
- return false;
- }
+ for (ReduceSinkOperator pRSNs : pRSs) {
+ ReduceSinkDesc pRSNc = pRSNs.getConf();
+ if (cRSc.getKeyCols().size() < pRSNc.getKeyCols().size()) {
+ return false;
+ }
+ if (cRSc.getPartitionCols().size() != pRSNc.getPartitionCols().size()) {
+ return false;
+ }
+ Integer moveReducerNumTo = checkNumReducer(cRSc.getNumReducers(), pRSNc.getNumReducers());
+ if (moveReducerNumTo == null ||
+ moveReducerNumTo > 0 && cRSc.getNumReducers() < minReducer) {
+ return false;
+ }
- Integer moveRSOrderTo = checkOrder(cRSc.getOrder(), pRS0c.getOrder());
- if (moveRSOrderTo == null) {
- return false;
+ Integer moveRSOrderTo = checkOrder(cRSc.getOrder(), pRSNc.getOrder());
+ if (moveRSOrderTo == null) {
+ return false;
+ }
}
boolean[] sorted = CorrelationUtilities.getSortedTags(pJoin);
@@ -231,11 +233,10 @@ public class ReduceSinkDeDuplication extends Transform {
}
}
- if (moveReducerNumTo > 0) {
- for (ReduceSinkOperator pRS : pRSs) {
- pRS.getConf().setNumReducers(cRS.getConf().getNumReducers());
- }
+ for (ReduceSinkOperator pRS : pRSs) {
+ pRS.getConf().setNumReducers(cRS.getConf().getNumReducers());
}
+
return true;
}