You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by vg...@apache.org on 2019/09/25 01:15:33 UTC
[hive] branch master updated: HIVE-22227: Tez bucket pruning
produces wrong result with shared work optimization (Vineet Garg,
reviewed by Jesus Camacho Rodriguez)
This is an automated email from the ASF dual-hosted git repository.
vgarg pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new c6875a3 HIVE-22227: Tez bucket pruning produces wrong result with shared work optimization (Vineet Garg, reviewed by Jesus Camacho Rodriguez)
c6875a3 is described below
commit c6875a378dd78f3d3b3c6876455e69fdc24d13e4
Author: Vineet Garg <vg...@apache.org>
AuthorDate: Tue Sep 24 18:14:52 2019 -0700
HIVE-22227: Tez bucket pruning produces wrong result with shared work optimization (Vineet Garg, reviewed by Jesus Camacho Rodriguez)
---
.../org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java | 7 +++++++
ql/src/test/queries/clientpositive/mergejoin.q | 2 ++
2 files changed, 9 insertions(+)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java
index 0e16b7b..f6b43db 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java
@@ -924,6 +924,12 @@ public class SharedWorkOptimizer extends Transform {
if (!prevTsOpPPList.getPartitions().equals(tsOpPPList.getPartitions())) {
return false;
}
+
+ if(!Objects.equals(tsOp1.getConf().getIncludedBuckets(),
+ tsOp2.getConf().getIncludedBuckets())) {
+ return false;
+ }
+
return true;
}
@@ -1456,6 +1462,7 @@ public class SharedWorkOptimizer extends Transform {
&& pctx.getPrunedPartitions(tsOp1).getPartitions().equals(
pctx.getPrunedPartitions(tsOp2).getPartitions())
&& op1Conf.getRowLimit() == op2Conf.getRowLimit()
+ && Objects.equals(op1Conf.getIncludedBuckets(), op2Conf.getIncludedBuckets())
&& Objects.equals(op1Conf.getOpProps(), op2Conf.getOpProps())) {
return true;
} else {
diff --git a/ql/src/test/queries/clientpositive/mergejoin.q b/ql/src/test/queries/clientpositive/mergejoin.q
index 8636f13..0da7eee 100644
--- a/ql/src/test/queries/clientpositive/mergejoin.q
+++ b/ql/src/test/queries/clientpositive/mergejoin.q
@@ -17,6 +17,8 @@ set hive.vectorized.execution.enabled=true;
set hive.tez.min.bloom.filter.entries=1;
set hive.tez.bigtable.minsize.semijoin.reduction=1;
+set hive.tez.bucket.pruning=true;
+
-- SORT_QUERY_RESULTS
explain vectorization detail